How does strange code hide errors? TensorFlow.NET Analysis

Introduction

TensorFlow.NET

Fragments that attracted attention when studying the analyzer report

Sophisticated collection traverse

private static void _RemoveDefaultAttrs(....)
{
var producer_op_dict = new Dictionary<string, OpDef>();
producer_op_list.Op.Select(op =>
{
producer_op_dict[op.Name] = op;
return op;
}).ToArray();
....
}
var producer_op_dict = new Dictionary<string, OpDef>();foreach (var op in producer_op_list.Op)
{
producer_op_dict[op.Name] = op;
}
public GraphDef convert_variables_to_constants(....)
{
....
inference_graph.Node.Select(x => map_name_to_node[x.Name] = x).ToArray();
....
}

Temporary solution

public GraphDef convert_variables_to_constants(....)
{
....
var source_op_name = get_input_name(node);
while(map_name_to_node[source_op_name].Op == "Identity")
{
throw new NotImplementedException);
....
}
....
}
public static Tensor[] _SoftmaxCrossEntropyWithLogitsGrad(
Operation op, Tensor[] grads
)
{
var grad_loss = grads[0];
var grad_grad = grads[1];
var softmax_grad = op.outputs[1];
var grad = _BroadcastMul(grad_loss, softmax_grad);
var logits = op.inputs[0];
if(grad_grad != null && !IsZero(grad_grad)) // <=
{
throw new NotImplementedException("_SoftmaxCrossEntropyWithLogitsGrad");
}
return new Tensor[]
{
grad,
_BroadcastMul(grad_loss, -nn_ops.log_softmax(logits))
};
}
private static bool IsZero(Tensor g)
{
if (new string[] { "ZerosLike", "Zeros" }.Contains(g.op.type))
return true;
throw new NotImplementedException("IsZero");
}

Is Tensor is Tensor?

private static Tensor[] _ExtractInputShapes(Tensor[] inputs)
{
var sizes = new Tensor[inputs.Length];
bool fully_known = true;
for(int i = 0; i < inputs.Length; i++)
{
var x = inputs[i];
var input_shape = array_ops.shape(x);
if (!(input_shape is Tensor) || input_shape.op.type != "Const")
{
fully_known = false;
break;
}
sizes[i] = input_shape;
}
....
}

Decent condition checking

public static Tensor[] _BaseFusedBatchNormGrad(....)
{
....
if (data_format == "NCHW") // <=
throw new NotImplementedException("");
var results = grad_fun(new FusedBatchNormParams
{
YBackprop = grad_y,
X = x,
Scale = scale,
ReserveSpace1 = pop_mean,
ReserveSpace2 = pop_var,
ReserveSpace3 = version == 2 ? op.outputs[5] : null,
Epsilon = epsilon,
DataFormat = data_format,
IsTraining = is_training
});
var (dx, dscale, doffset) = (results[0], results[1], results[2]);
if (data_format == "NCHW") // <=
throw new NotImplementedException("");
....
}

The illusion of choice

public Tensor Activate(Tensor x, string name = null)
{
....
Tensor negative_part;
if (Math.Abs(_threshold) > 0.000001f)
{
negative_part = gen_ops.relu(-x + _threshold);
} else
{
negative_part = gen_ops.relu(-x + _threshold);
}
....
}
private static Operation _GroupControlDeps(
string dev, Operation[] deps, string name = null
)
{
return tf_with(ops.control_dependencies(deps), ctl =>
{
if (dev == null)
{
return gen_control_flow_ops.no_op(name);
}
else
{
return gen_control_flow_ops.no_op(name);
}
});
}

Late check

public static Tensor[] Input(int[] batch_shape = null,
TF_DataType dtype = TF_DataType.DtInvalid,
string name = null,
bool sparse = false,
Tensor tensor = null)
{
var batch_size = batch_shape[0];
var shape = batch_shape.Skip(1).ToArray(); // <=
InputLayer input_layer = null;
if (batch_shape != null) // <=
....
else
....
....
}

Another ″for later″ fragment?

public MnistDataSet(
NDArray images, NDArray labels, Type dataType, bool reshape // <=
)
{
EpochsCompleted = 0;
IndexInEpoch = 0;
NumOfExamples = images.shape[0]; images = images.reshape(
images.shape[0], images.shape[1] * images.shape[2]
);
images = images.astype(dataType);
// for debug np.multiply performance
var sw = new Stopwatch();
sw.Start();
images = np.multiply(images, 1.0f / 255.0f);
sw.Stop();
Console.WriteLine($"{sw.ElapsedMilliseconds}ms");
Data = images;
labels = labels.astype(dataType);
Labels = labels;
}

Uncatchable possible null dereference

public static Tensor[] _GatherV2Grad(Operation op, Tensor[] grads)
{
....
if((int)axis_static == 0)
{
var params_tail_shape = params_shape.slice(new NumSharp.Slice(start:1));
var values_shape = array_ops.concat(
new[] { indices_size, params_tail_shape }, 0
);
var values = array_ops.reshape(grad, values_shape);
indices = array_ops.reshape(indices, indices_size);
return new Tensor[]
{
new IndexedSlices(values, indices, params_shape), // <=
null,
null
};
}
....
}
public IndexedSlices(
Tensor values, Tensor indices, Tensor dense_shape = null
)
{
_values = values;
_indices = indices;
_dense_shape = dense_shape;
_values.Tag = this; // <=
}
public static Tensor reshape<T1, T2>(T1 tensor, T2 shape, string name = null)
=> gen_array_ops.reshape(tensor, shape, null);
public static Tensor reshape<T1, T2>(T1 tensor, T2 shape, string name = null)
{
var _op = _op_def_lib._apply_op_helper(
"Reshape", name, new { tensor, shape }
);
return _op.output;
}
public Tensor output => _outputs.FirstOrDefault();

Unreliable waiting?

private (LoopVar<TItem>, Tensor[]) _BuildLoop<TItem>(
....
) where ....
{
....
// Finds the closest enclosing non-None control pivot.
var outer_context = _outer_context;
object control_pivot = null;
while (outer_context != null && control_pivot == null) // <=
{
} if (control_pivot != null)
{
}
....
}

Violation of bounds

public TensorShape(int[][] dims)
{
if(dims.Length == 1)
{
switch (dims[0].Length)
{
case 0: shape = new Shape(new int[0]); break;
case 1: shape = Shape.Vector((int)dims[0][0]); break;
case 2: shape = Shape.Matrix(dims[0][0], dims[1][2]); break; // <=
default: shape = new Shape(dims[0]); break;
}
}
else
{
throw new NotImplementedException("TensorShape int[][] dims");
}
}

A typo?

private void _init_from_args(object initial_value = null, ....) // <=
{
var init_from_fn = initial_value.GetType().Name == "Func`1"; // <=
....
tf_with(...., scope =>
{
....
tf_with(...., delegate
{
initial_value = ops.convert_to_tensor( // <=
init_from_fn ? (initial_value as Func<Tensor>)():initial_value,
name: "initial_value",
dtype: dtype
);
});
_shape = shape ?? (initial_value as Tensor).TensorShape;
_initial_value = initial_value as Tensor; // <=
....
_dtype = _initial_value.dtype.as_base_dtype(); // <=
if (_in_graph_mode)
{
....
if (initial_value != null) // <=
{
....
}
....
}
....
});
}
[DebuggerStepThrough] // with "Just My Code" enabled this lets the 
[DebuggerNonUserCode()] //debugger break at the origin of the exception
public static void tf_with<T>(
T py, Action<T> action
) where T : ITensorFlowObject
{
try
{
py.__enter__();
action(py);
}
finally
{
py.__exit__();
py.Dispose();
}
}

Conclusion

--

--

Get the Medium app

A button that says 'Download on the App Store', and if clicked it will lead you to the iOS App store
A button that says 'Get it on, Google Play', and if clicked it will lead you to the Google Play store