Add Activations

tensorflow-core/tensorflow-core-api/src/bazel/api_def/api_def_LeakyRelu.pbtxt

@@ -1,5 +1,6 @@
 op {
  graph_op_name: "LeakyRelu"
+ visibility: VISIBLE
  endpoint {
  name: "nn.LeakyRelu"
  }

tensorflow-core/tensorflow-core-api/src/gen/annotations/org/tensorflow/op/NnOps.java

@@ -59,6 +59,7 @@
 import org.tensorflow.op.nn.FusedResizeAndPadConv2d;
 import org.tensorflow.op.nn.InTopK;
 import org.tensorflow.op.nn.L2Loss;
+import org.tensorflow.op.nn.LeakyRelu;
 import org.tensorflow.op.nn.LearnedUnigramCandidateSampler;
 import org.tensorflow.op.nn.LocalResponseNormalization;
 import org.tensorflow.op.nn.LogSoftmax;
@@ -1226,6 +1227,19 @@ public <T extends TNumber> L2Loss<T> l2Loss(Operand<T> t) {
  return L2Loss.create(scope, t);
  }
 
+ /**
+ * Computes rectified linear: `max(features, features * alpha)`.
+ *
+ * @param <T> data type for {@code activations()} output
+ * @param features
+ * @param options carries optional attributes values
+ * @return a new instance of LeakyRelu
+ */
+ public <T extends TNumber> LeakyRelu<T> leakyRelu(Operand<T> features,
+ LeakyRelu.Options... options) {
+ return LeakyRelu.create(scope, features, options);
+ }
+
  /**
  * Generates labels for candidate sampling with a learned unigram distribution.
  * <p>

tensorflow-core/tensorflow-core-api/src/gen/java/org/tensorflow/op/nn/LeakyRelu.java

@@ -33,6 +33,7 @@
  * 
  * @param <T> data type for {@code activations()} output
  */
+@Operator(group = "nn")
 public final class LeakyRelu<T extends TNumber> extends RawOp implements Operand<T> {
 
  /**

tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/types/TBfloat16.java

@@ -30,7 +30,7 @@
 import org.tensorflow.ndarray.NdArray;
 import org.tensorflow.ndarray.StdArrays;
 import org.tensorflow.ndarray.impl.dense.FloatDenseNdArray;
-import org.tensorflow.types.family.TNumber;
+import org.tensorflow.types.family.TFloating;
 
 /**
  * Brain 16-bit float tensor type.
@@ -48,7 +48,7 @@
  * <p>Note that some CPUs support the bfloat16 format natively, which can result in faster
  * computation compared to {@link TFloat16} when GPUs are not used.
  */
-public interface TBfloat16 extends FloatNdArray, TNumber {
+public interface TBfloat16 extends FloatNdArray, TFloating {
  /** readable-name for the data type */
  static final String NAME = "BFLOAT16";
 

tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/types/TFloat16.java

@@ -30,7 +30,7 @@
 import org.tensorflow.ndarray.NdArray;
 import org.tensorflow.ndarray.StdArrays;
 import org.tensorflow.ndarray.impl.dense.FloatDenseNdArray;
-import org.tensorflow.types.family.TNumber;
+import org.tensorflow.types.family.TFloating;
 
 /**
  * IEEE-754 half-precision 16-bit float tensor type.
@@ -45,7 +45,7 @@
  * most CPUs do not support this format natively. For CPU computation on 16-bit floats, the {@link
  * TBfloat16} tensor type might be a better option.
  */
-public interface TFloat16 extends FloatNdArray, TNumber {
+public interface TFloat16 extends FloatNdArray, TFloating {
 
  /** readable-name for the data type */
  static final String NAME = "FLOAT16";

tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/types/TFloat32.java

@@ -29,10 +29,10 @@
 import org.tensorflow.ndarray.NdArray;
 import org.tensorflow.ndarray.StdArrays;
 import org.tensorflow.ndarray.impl.dense.FloatDenseNdArray;
-import org.tensorflow.types.family.TNumber;
+import org.tensorflow.types.family.TFloating;
 
 /** IEEE-754 single-precision 32-bit float tensor type. */
-public interface TFloat32 extends FloatNdArray, TNumber {
+public interface TFloat32 extends FloatNdArray, TFloating {
 
  /** readable-name for the data type */
  static final String NAME = "FLOAT";

tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/types/TFloat64.java

@@ -29,10 +29,11 @@
 import org.tensorflow.ndarray.NdArray;
 import org.tensorflow.ndarray.StdArrays;
 import org.tensorflow.ndarray.impl.dense.DoubleDenseNdArray;
-import org.tensorflow.types.family.TNumber;
+import org.tensorflow.types.family.TFloating;
+
 
 /** IEEE-754 double-precision 64-bit float tensor type. */
-public interface TFloat64 extends DoubleNdArray, TNumber {
+public interface TFloat64 extends DoubleNdArray, TFloating {
 
  /** readable-name for the data type */
  static final String NAME = "DOUBLE";

tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/types/family/TFloating.java

@@ -0,0 +1,19 @@
+package org.tensorflow.types.family;
+
+/**
+ * Marker interface for floating point tensor types.
+ *
+ * <p>Operations that only accepts floating point values as some of their operands enforce that the tensor
+ * types for these operands to be bound to this interface. For example:
+ *
+ * <pre>{@code
+ * TFloat32 tensor1 = TFloat32.vectorOf(1, 2, 3);
+ * TBool tensor2 = TBool.vectorOf(true, false, true);
+ *
+ * Ops tf = Ops.create();
+ * Exponential<TFloat32> exp = new Exponential<>(tf);
+ * exp.call(tf.constant(tensor1)); // OK
+ * exp.call(tf.constant(tensor2)); // Compilation failure
+ * }</pre>
+ */
+public interface TFloating extends TNumber {}

tensorflow-framework/src/main/java/org/tensorflow/framework/activations/Activation.java

@@ -0,0 +1,68 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=======================================================================*/
+package org.tensorflow.framework.activations;
+
+import org.tensorflow.Operand;
+import org.tensorflow.op.Ops;
+import org.tensorflow.types.family.TNumber;
+
+/**
+ * Abstract base class for Activations
+ *
+ * <p><b>Note:</b> The {@link #tf} attribute must be set prior to invoking the call method. See
+ * {@link #setTF(Ops)} and the constructor {@link #Activation(Ops)}.
+ *
+ * @param <T> the data type of the activation
+ */
+public abstract class Activation<T extends TNumber> {
+
+ /** The TensorFlow Ops */
+ protected Ops tf;
+
+ /**
+ * Creates the abstract class for an Activation
+ *
+ * @param tf the TensorFlow Ops
+ */
+ protected Activation(Ops tf) {
+ this.tf = tf;
+ }
+
+ /**
+ * Sets the TensorFlow Ops
+ *
+ * @param tf the TensorFlow Ops
+ */
+ protected void setTF(Ops tf) {
+ this.tf = tf;
+ }
+
+ /**
+ * Gets the TensorFlow Ops
+ *
+ * @return the TensorFlow Ops
+ */
+ protected Ops getTF() {
+ return this.tf;
+ }
+
+ /**
+ * Gets the calculation operation for the activation.
+ *
+ * @param input the input tensor
+ * @return The operand for the activation
+ */
+ public abstract Operand<T> call(Operand<T> input);
+}

tensorflow-framework/src/main/java/org/tensorflow/framework/activations/ELU.java

@@ -0,0 +1,98 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=======================================================================*/
+package org.tensorflow.framework.activations;
+
+import org.tensorflow.DataType;
+import org.tensorflow.Operand;
+import org.tensorflow.op.Ops;
+import org.tensorflow.types.TBool;
+import org.tensorflow.types.family.TFloating;
+
+/**
+ * Exponential linear unit.
+ *
+ * <p>The exponential linear unit (ELU) with <code>alpha &gt; 0</code> is:
+ *
+ * <p><code>x</code> if <code>x &gt; 0</code> and <code>alpha * (exp(x) -
+ * 1)</code> if <code>x &lt; 0</code>.
+ *
+ * <p>The ELU hyperparameter <code>alpha</code> controls the value to which an ELU saturates for
+ * negative net inputs. ELUs diminish the vanishing gradient effect.
+ *
+ * <p>ELUs have negative values which pushes the mean of the activations closer to zero. Mean
+ * activations that are closer to zero enable faster learning as they bring the gradient closer to
+ * the natural gradient. ELUs saturate to a negative value when the argument gets smaller.
+ * Saturation means a small derivative which decreases the variation and the information that is
+ * propagated to the next layer.
+ *
+ * <p>Example Usage:
+ *
+ * <pre>
+ * Operand&lt;TFloat32&gt; input = &#46;&#46;&#46;;
+ * ELU&lt;TFloat32&gt; elu = new ELU&lt;&gt;(tf, 2.0f);
+ * Operand&lt;TFloat32&gt; result = elu.call(input);
+ * </pre>
+ *
+ * @param <T> the data type of the activation
+ * @see <a href="https://arxiv.org/abs/1511.07289">Clevert et al, 2016, Fast and Accurate Deep
+ * Network Learning by Exponential Linear Units (ELUs)</a>
+ */
+public class ELU<T extends TFloating> extends Activation<T> {
+
+ private static final double ALPHA_DEFAULT = 1.0;
+
+ /** A scalar, slope of negative section. */
+ private final double alpha;
+
+ /**
+ * Creates a new ELU with alpha={@link #ALPHA_DEFAULT}.
+ *
+ * @param tf the TensorFlow Ops
+ */
+ public ELU(Ops tf) {
+ this(tf, ALPHA_DEFAULT);
+ }
+
+ /**
+ * Creates a new ELU
+ *
+ * @param tf the TensorFlow Ops
+ * @param alpha A scalar, slope of negative section. It controls the value to which an ELU
+ * saturates for negative net inputs.
+ */
+ public ELU(Ops tf, double alpha) {
+ super(tf);
+ this.alpha = alpha;
+ }
+
+ /**
+ * Gets the calculation operation for the activation.
+ *
+ * @param input the input tensor
+ * @return The operand for the activation
+ */
+ @Override
+ public Operand<T> call(Operand<T> input) {
+
+ Operand<T> result = tf.nn.elu(input);
+ if (alpha == 1.0) return result;
+ else {
+ DataType<T> dataType = input.asOutput().dataType();
+ Operand<T> y = tf.math.mul(result, tf.dtypes.cast(tf.constant(alpha), dataType));
+ Operand<TBool> cond = tf.math.greater(result, tf.dtypes.cast(tf.constant(0), dataType));
+ return tf.select(cond, result, y);
+ }
+ }
+}

tensorflow-framework/src/main/java/org/tensorflow/framework/activations/Exponential.java

@@ -0,0 +1,57 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=======================================================================*/
+package org.tensorflow.framework.activations;
+
+import org.tensorflow.Operand;
+import org.tensorflow.op.Ops;
+import org.tensorflow.types.family.TFloating;
+
+/**
+ * Exponential activation function.
+ *
+ * <p>For example:
+ *
+ * <pre>
+ * Operand&lt;TFloat32&gt; input = tf.constant(
+ * new float[] {-3.0f,-1.0f, 0.0f,1.0f,3.0f});
+ * Exponential&lt;TFloat32&gt; exp = new Exponential&lt;&gt;(tf);
+ * Operand&lt;TFloat32&gt; result = exp.call(input);
+ * // result is [0.04978707f, 0.36787945f, 1.f, 2.7182817f, 20.085537f]
+ * </pre>
+ *
+ * @param <T> the data type of the activation
+ */
+public class Exponential<T extends TFloating> extends Activation<T> {
+
+ /**
+ * Creates an Exponential activation.
+ *
+ * @param tf the TensorFlow Ops
+ */
+ public Exponential(Ops tf) {
+ super(tf);
+ }
+
+ /**
+ * Calculates the Exponential activation.
+ *
+ * @param input the input tensor
+ * @return an Operand for the exponential activation: <code>exp(x)</code>.
+ */
+ @Override
+ public Operand<T> call(Operand<T> input) {
+ return tf.math.exp(input);
+ }
+}