Constraints - 1

tensorflow-core/tensorflow-core-api/src/gen/annotations/org/tensorflow/op/Ops.java

@@ -1834,13 +1834,14 @@ public Constant<TInt32> constant(Shape shape, IntDataBuffer data) {
   }
 
   /**
-   * Creates a scalar of {@code type}, with the value of {@code number}.
-   *  {@code number} may be truncated if it does not fit in the target type.
+   * Creates a scalar of {@code type}, with the value of {@code number}. {@code number} may be truncated if it does not
+   *  fit in the target type.
    *
    * @param type the type of tensor to create.  Must be concrete (i.e. not {@link org.tensorflow.types.family.TFloating})
    * @param number the value of the tensor
    * @return a constant of the passed type
-   * @throws IllegalArgumentException if the type is abstract (i.e. {@link org.tensorflow.types.family.TFloating}) or unknown.
+   * @throws IllegalArgumentException if the type is abstract (i.e. {@link org.tensorflow.types.family.TFloating}) or
+   *  unknown.
    */
   public <T extends TNumber> Constant<T> constant(Class<T> type, Number number) {
     return Constant.tensorOf(scope, type, number);
@@ -1892,14 +1893,14 @@ public <T extends TType> Constant<T> constantOf(T tensor) {
   }
 
   /**
-   * Creates a scalar of the same type as {@code toMatch}, with the value of {@code number}.
-   *  {@code number} may be truncated if it does not fit in the target type.
+   * Creates a scalar of the same type as {@code toMatch}, with the value of {@code number}. {@code number} may be
+   *  truncated if it does not fit in the target type.
    *
    * @param toMatch the operand providing the target type
    * @param number the value of the tensor
    * @return a constant with the same type as {@code toMatch}
-   * @see Ops#constant(Class, Number)
    * @throws IllegalArgumentException if the type is unknown (which should be impossible).
+   * @see Ops#constant(Class, Number)
    */
   public <T extends TNumber> Constant<T> constantOfSameType(Operand<T> toMatch, Number number) {
     return Constant.tensorOfSameType(scope, toMatch, number);

tensorflow-framework/src/main/java/org/tensorflow/framework/constraints/Constraint.java

@@ -0,0 +1,110 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=======================================================================*/
+package org.tensorflow.framework.constraints;
+
+import org.tensorflow.Operand;
+import org.tensorflow.op.Ops;
+import org.tensorflow.op.core.ReduceSum;
+import org.tensorflow.types.family.TNumber;
+
+import static org.tensorflow.framework.utils.CastHelper.cast;
+
+/** Base class for Constraints. Constraint subclasses impose constraints on weight values */
+public abstract class Constraint {
+
+  public static final float EPSILON = 1e-7f;
+
+  private final Ops tf;
+
+  /**
+   * Creates a Constraint
+   *
+   * @param tf the TensorFlow Ops
+   */
+  public Constraint(Ops tf) {
+    this.tf = tf;
+  }
+
+  /**
+   * Applies the constraint against the provided weights
+   *
+   * @param weights the weights
+   * @return the constrained weights
+   */
+  public abstract <T extends TNumber> Operand<T> call(Operand<T> weights);
+
+  /**
+   * Gets the TensorFlow Ops
+   *
+   * @return the TensorFlow Ops
+   */
+  public Ops getTF() {
+    return tf;
+  }
+
+  /**
+   * Gets the element-wise square root.
+   *
+   * @param x the input Operand.
+   * @return the element-wise square root.
+   * @param <T> The data type for the operand and result.
+   * @throws IllegalArgumentException if x is null
+   */
+  protected <T extends TNumber> Operand<T> sqrt(Operand<T> x) {
+    if (x == null) throw new IllegalArgumentException("Operand x must not be null");
+    Class<T> type = x.type();
+    Operand<T> zero = cast(tf, tf.constant(0), type);
+    Operand<T> inf = cast(tf, tf.constant(Double.POSITIVE_INFINITY), type);
+    return tf.math.sqrt(tf.clipByValue(x, zero, inf));
+  }
+
+  /**
+   * Gets the element-wise value clipping.
+   *
+   * @param x the Operand to clip
+   * @param minValue the minimum value
+   * @param maxValue the maximum value
+   * @return the operand with clipped values
+   * @param <T> The data type for the operand and result.
+   * @throws IllegalArgumentException if x is null
+   */
+  protected <T extends TNumber> Operand<T> clip(Operand<T> x, double minValue, double maxValue) {
+    if (x == null) throw new IllegalArgumentException("Operand x must not be null");
+    Ops tf = getTF();
+    Class<T> type = x.type();
+
+    double min = Math.min(minValue, maxValue);
+    double max = Math.max(minValue, maxValue);
+
+    Operand<T> minValueConstant = cast(tf, tf.constant(min), type);
+    Operand<T> maxValueConstant = cast(tf, tf.constant(max), type);
+    return tf.clipByValue(x, minValueConstant, maxValueConstant);
+  }
+
+  /**
+   * Calculates the norm of the weights along the axes
+   *
+   * @param weights the weights used to calculate the norms
+   * @param axes the axes along which to calculate weight norms.
+   * @param <T> the data type for the weights and the result
+   * @return the norms
+   * @throws IllegalArgumentException if weights is null
+   */
+  protected <T extends TNumber> Operand<T> norm(Operand<T> weights, int[] axes) {
+    if (weights == null) throw new IllegalArgumentException("weights must not be null");
+    return sqrt(
+        tf.reduceSum(tf.math.square(weights), tf.constant(axes), ReduceSum.keepDims(Boolean.TRUE)));
+  }
+}

tensorflow-framework/src/main/java/org/tensorflow/framework/constraints/MaxNorm.java

@@ -0,0 +1,109 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=======================================================================*/
+package org.tensorflow.framework.constraints;
+
+import org.tensorflow.Operand;
+import org.tensorflow.op.Ops;
+import org.tensorflow.types.family.TNumber;
+
+import static org.tensorflow.framework.utils.CastHelper.cast;
+
+/**
+ * Constrains the weights incident to each hidden unit to have a norm less than or equal to a
+ * desired value.
+ */
+public class MaxNorm extends Constraint {
+  public static final double MAX_VALUE_DEFAULT = 2.0;
+  public static final int AXIS_DEFAULT = 0;
+
+  /** the maximum norm for the incoming weights. */
+  private final double maxValue;
+  /** integer, axis along which to calculate weight norms. */
+  private final int[] axes;
+
+  /**
+   * Create a MaxNorm constraint using {@link #MAX_VALUE_DEFAULT} for the max value and {@link
+   * #AXIS_DEFAULT} for the axis.
+   *
+   * @param tf the TensorFlow Ops
+   */
+  public MaxNorm(Ops tf) {
+    this(tf, MAX_VALUE_DEFAULT, AXIS_DEFAULT);
+  }
+
+  /**
+   * Create a MaxNorm constraint using {@link #AXIS_DEFAULT} for the axis.
+   *
+   * @param tf the TensorFlow Ops
+   * @param maxValue the maximum norm for the incoming weights.
+   */
+  public MaxNorm(Ops tf, double maxValue) {
+    this(tf, maxValue, AXIS_DEFAULT);
+  }
+
+  /**
+   * Create a MaxNorm constraint
+   *
+   * @param tf the TensorFlow Ops
+   * @param maxValue the maximum norm for the incoming weights.
+   * @param axis axis along which to calculate weight norms.
+   */
+  public MaxNorm(Ops tf, double maxValue, int axis) {
+    this(tf, maxValue, new int[] {axis});
+  }
+
+  /**
+   * Create a MaxNorm constraint
+   *
+   * @param tf the TensorFlow Ops
+   * @param maxValue the maximum norm for the incoming weights.
+   * @param axes axes along which to calculate weight norms.
+   */
+  public MaxNorm(Ops tf, double maxValue, int[] axes) {
+    super(tf);
+    this.maxValue = maxValue;
+    this.axes = axes;
+  }
+
+  /** {@inheritDoc} */
+  @Override
+  public <T extends TNumber> Operand<T> call(Operand<T> weights) {
+    Ops tf = getTF();
+    Class<T> type = weights.type();
+    Operand<T> norms = norm(weights, getAxes());
+    Operand<T> desired = clip(norms, 0f, this.getMaxValue());
+
+    return tf.math.mul(
+        weights, tf.math.div(desired, tf.math.add(cast(tf, tf.constant(EPSILON), type), norms)));
+  }
+
+  /**
+   * Gets the max value
+   *
+   * @return the maxValue
+   */
+  public double getMaxValue() {
+    return maxValue;
+  }
+
+  /**
+   * Gets the axes
+   *
+   * @return the axes
+   */
+  public int[] getAxes() {
+    return axes;
+  }
+}