Merge pull request #18 from rcrowder/release-1_2_1

Release 1.2.1

CHANGES.md

@@ -1,3 +1,10 @@
+1.2.1  December 22, 2016
+========================
+
+- Projection fix in Kernel
+- Chunk encoder uses distance metric
+- Better default parameters
+
 1.2  December 19, 2016
 ========================
 

README.md

@@ -156,7 +156,6 @@ Sparse features Chunk (prefix 'sfc'):
  - sfc_chunkSize (int, int): Size of a chunk.
  - sfc_initWeightRange (float, float): Weight initialization range.
  - sfc_numSamples (int): Number of temporally extended samples (1 means no additional samples).
- - sfc_gamma (float): Falloff (higher means faster) for SOM neighborhood radius.
  - Feed forward inputs (prefix 'ff'):
     - sfc_ff_radius (int): Radius onto feed forward inputs.
     - sfc_ff_weightAlpha (float): Learning rate for feed forward inputs.

doxygen_config

@@ -38,7 +38,7 @@ PROJECT_NAME           = OgmaNeo
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = 1.2
+PROJECT_NUMBER         = 1.2.1
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a

resources/kernels/neoKernelsAgentSwarm.cl

@@ -15,7 +15,7 @@ void kernel alDeriveInputs(read_only image2d_t inputs, read_only image2d_t outpu
 
 	float outputPrev = read_imagef(outputsBack, defaultSampler, position).x;
 
-    write_imagef(outputsFront, position, (float4)(input, fabs(input - outputPrev), 0.0f, 0.0f));
+    write_imagef(outputsFront, position, (float4)(input, fmax(0.0f, input - outputPrev), 0.0f, 0.0f));
 }
 
 void kernel alActivate(read_only image2d_t visibleStates,

resources/kernels/neoKernelsPredictor.cl

@@ -15,7 +15,7 @@ void kernel plDeriveInputs(read_only image2d_t inputs, read_only image2d_t outpu
 
 	float outputPrev = read_imagef(outputsBack, defaultSampler, position).x;
 
-    write_imagef(outputsFront, position, (float4)(input, fabs(input - outputPrev), 0.0f, 0.0f));
+    write_imagef(outputsFront, position, (float4)(input, fmax(0.0f, input - outputPrev), 0.0f, 0.0f));
 }
 
 void kernel plStimulus(read_only image2d_t visibleStates,

resources/kernels/neoKernelsSparseFeaturesChunk.cl

@@ -30,9 +30,9 @@ void kernel sfcStimulus(read_only image3d_t samples,
 	int2 hiddenPosition = (int2)(get_global_id(0), get_global_id(1));
 
 	int2 chunkPosition = (int2)(hiddenPosition.x / chunkSize.x, hiddenPosition.y / chunkSize.y);
-	int2 chunkCenter = (int2)(chunkPosition.x + 0.5f, chunkPosition.y + 0.5f);
+	float2 chunkCenter = (float2)(chunkPosition.x + 0.5f, chunkPosition.y + 0.5f);
 
-	int2 visiblePositionCenter = project(chunkCenter, chunkToVisible);
+	int2 visiblePositionCenter = projectf(chunkCenter, chunkToVisible);
 
 	float sum = read_imagef(hiddenSummationTempBack, defaultSampler, hiddenPosition).x;
 
@@ -57,7 +57,9 @@ void kernel sfcStimulus(read_only image3d_t samples,
 
 					float sample = read_imagef(samples, defaultSampler, (int4)(visiblePosition.x, visiblePosition.y, s, 0)).x;
 
-					subSum += sample * weight;
+					float delta = sample - weight;
+
+					subSum += -delta * delta;
 				}
 			}
 	}
@@ -72,7 +74,7 @@ void kernel sfcActivate(read_only image2d_t hiddenStimuli, read_only image2d_t h
 
 	float hiddenStimulus = read_imagef(hiddenStimuli, defaultSampler, hiddenPosition).x;
 
-	float tracePrev = read_imagef(hiddenStatesPrev, defaultSampler, hiddenPosition).y;
+	//float tracePrev = read_imagef(hiddenStatesPrev, defaultSampler, hiddenPosition).y;
 
 	write_imagef(hiddenActivationsFront, hiddenPosition, (float4)(hiddenStimulus, 0.0f, 0.0f, 0.0f));
 }
@@ -113,6 +115,8 @@ void kernel sfcInhibit(read_only image2d_t activations,
 			if (inBounds0(hiddenPosition, hiddenSize)) {
 				float hiddenState = (dx == maxDelta.x && dy == maxDelta.y) ? 1.0f : 0.0f;
 
+				//float tracePrev = read_imagef(hiddenStatesBack, defaultSampler, hiddenPosition).y;
+
 				write_imagef(hiddenStatesFront, hiddenPosition, (float4)(hiddenState, 0.0f, 0.0f, 0.0f));
 			}
 		}
@@ -164,9 +168,9 @@ void kernel sfcLearnWeights(read_only image2d_t chunkWinners, read_only image2d_
 	int2 hiddenPosition = (int2)(get_global_id(0), get_global_id(1));
 
 	int2 chunkPosition = (int2)(hiddenPosition.x / chunkSize.x, hiddenPosition.y / chunkSize.y);
-	int2 chunkCenter = (int2)(chunkPosition.x + 0.5f, chunkPosition.y + 0.5f);
+	float2 chunkCenter = (float2)(chunkPosition.x + 0.5f, chunkPosition.y + 0.5f);
 
-	int2 visiblePositionCenter = project(chunkCenter, chunkToVisible);
+	int2 visiblePositionCenter = projectf(chunkCenter, chunkToVisible);
 
 	int2 fieldLowerBound = visiblePositionCenter - (int2)(radius);
 
@@ -186,28 +190,8 @@ void kernel sfcLearnWeights(read_only image2d_t chunkWinners, read_only image2d_
 	float neighbor = (manhattanDist == 1 || (abs(delta.x) == 1 && abs(delta.y) == 1)) ? 1.0f : 0.0f;
 
 	float strength = (chunkWinnerf.x == chunkWinnerPrevf.x && chunkWinnerf.y == chunkWinnerPrevf.y) ? 0.0f : hiddenState + neighbor;
-
-	float weightSum = 0.0f;
-
-	for (int s = 0; s < numSamples; s++) {
-		for (int dx = -radius; dx <= radius; dx++)
-			for (int dy = -radius; dy <= radius; dy++) {
-				int2 visiblePosition = visiblePositionCenter + (int2)(dx, dy);
-
-				if (inBounds0(visiblePosition, visibleSize)) {
-					int2 offset = visiblePosition - fieldLowerBound;
+	//float strength = hiddenState + neighbor;
 
-					int wi = s + numSamples * (offset.y + offset.x * (radius * 2 + 1));
-
-					float weightPrev = read_imagef(weightsBack, defaultSampler, (int4)(hiddenPosition.x, hiddenPosition.y, wi, 0)).x;
-
-					weightSum += weightPrev * weightPrev;
-				}
-			}
-	}
-
-	float scale = 1.0f / fmax(0.0001f, weightSum);
-
 	for (int s = 0; s < numSamples; s++) {
 		for (int dx = -radius; dx <= radius; dx++)
 			for (int dy = -radius; dy <= radius; dy++) {
@@ -222,9 +206,11 @@ void kernel sfcLearnWeights(read_only image2d_t chunkWinners, read_only image2d_
 
 					float sample = read_imagef(samples, defaultSampler, (int4)(visiblePosition.x, visiblePosition.y, s, 0)).x;
 
-					float sLearn = strength * sample; 
+					float delta = sample - weightPrev;
+
+					float sLearn = strength * delta; 
 
-					write_imagef(weightsFront, (int4)(hiddenPosition.x, hiddenPosition.y, wi, 0), (float4)(weightPrev * scale + weightAlpha * sLearn, 0.0f, 0.0f, 0.0f));
+					write_imagef(weightsFront, (int4)(hiddenPosition.x, hiddenPosition.y, wi, 0), (float4)(weightPrev + weightAlpha * sLearn, 0.0f, 0.0f, 0.0f));
 				}
 			}
 	}

resources/kernels/neoKernelsSparseFeaturesReLU.cl

@@ -218,9 +218,7 @@ void kernel sfrLearnWeightsHidden(read_only image2d_t hiddenStatesPrev,
 	float2 hiddenStatePrev = read_imagef(hiddenStatesPrev, defaultSampler, hiddenPosition).xy;
 
 	float error = (hiddenStatePrev.x == 0.0f ? 0.0f : 1.0f - hiddenStatePrev.x * hiddenStatePrev.x) * (read_imagef(errors, defaultSampler, hiddenPosition).x > 0.0f ? 1.0f : -1.0f);
-
-	error *= (error > 0.0f ? (1.0f - hiddenStatePrev.y) : hiddenStatePrev.y);
-
+
 	for (int s = 0; s < numSamples; s++) {
 		for (int dx = -radius; dx <= radius; dx++)
 			for (int dy = -radius; dy <= radius; dy++) {
@@ -287,11 +285,9 @@ void kernel sfrLearnBiases(read_only image2d_t hiddenStatesPrev,
 
 	float error = (hiddenStatePrev.x == 0.0f ? 0.0f : 1.0f - hiddenStatePrev.x * hiddenStatePrev.x) * (read_imagef(errors, defaultSampler, hiddenPosition).x > 0.0f ? 1.0f : -1.0f);
 
-	error *= (error > 0.0f ? (1.0f - hiddenStatePrev.y) : hiddenStatePrev.y);
-
 	float biasPrev = read_imagef(biasesBack, defaultSampler, hiddenPosition).x;
 
-	float bias = biasPrev + biasAlpha * error;
+	float bias = biasPrev + biasAlpha * (activeRatio - hiddenStatePrev.x);
 
 	write_imagef(biasesFront, hiddenPosition, (float4)(bias, 0.0f, 0.0f, 0.0f));
 }

resources/kernels/neoKernelsSparseFeaturesSTDP.cl

@@ -39,11 +39,11 @@ void kernel sfsStimulus(read_only image2d_t visibleStates,
                 float visibleState = read_imagef(visibleStates, defaultSampler, visiblePosition).x;
 
                 subSum += weight * visibleState;
-				count += visibleState * visibleState;
+				count += weight;
             }
         }
 
-    write_imagef(hiddenSummationTempFront, hiddenPosition, (float4)(sum + subSum / fmax(0.0001f, sqrt(count)), 0.0f, 0.0f, 0.0f));
+    write_imagef(hiddenSummationTempFront, hiddenPosition, (float4)(sum + subSum / fmax(0.0001f, count), 0.0f, 0.0f, 0.0f));
 }
 
 void kernel sfsActivate(read_only image2d_t stimuli, read_only image2d_t hiddenStatesPrev, read_only image2d_t biases,
@@ -146,25 +146,6 @@ void kernel sfsLearnWeights(read_only image2d_t hiddenStates, read_only image2d_
     float hiddenState = read_imagef(hiddenStates, defaultSampler, hiddenPosition).x;
     float hiddenStatePrev = read_imagef(hiddenStatesPrev, defaultSampler, hiddenPosition).x;
 
-	float weightSum = 0.0f;
-
-	for (int dx = -radius; dx <= radius; dx++)
-        for (int dy = -radius; dy <= radius; dy++) {
-            int2 visiblePosition = visiblePositionCenter + (int2)(dx, dy);
-
-            if (inBounds0(visiblePosition, visibleSize)) {
-                int2 offset = visiblePosition - fieldLowerBound;
-
-                int wi = offset.y + offset.x * (radius * 2 + 1);
-
-                float weightPrev = read_imagef(weightsBack, defaultSampler, (int4)(hiddenPosition.x, hiddenPosition.y, wi, 0)).x;
-
-                weightSum += weightPrev * weightPrev;
-            }
-        }
-
-	float scale = 1.0f / fmax(0.0001f, sqrt(weightSum));
-
     for (int dx = -radius; dx <= radius; dx++)
         for (int dy = -radius; dy <= radius; dy++) {
             int2 visiblePosition = visiblePositionCenter + (int2)(dx, dy);
@@ -179,9 +160,9 @@ void kernel sfsLearnWeights(read_only image2d_t hiddenStates, read_only image2d_
                 float visibleState = read_imagef(visibleStates, defaultSampler, visiblePosition).x;
 				float visibleStatePrev = read_imagef(visibleStatesPrev, defaultSampler, visiblePosition).x;
 
-                float learn = hiddenStatePrev * visibleStatePrev - hiddenStatePrev * visibleState;
+                float learn = hiddenStatePrev * visibleStatePrev * (1.0f - weightPrev) - hiddenStatePrev * visibleState * weightPrev;
 
-                write_imagef(weightsFront, (int4)(hiddenPosition.x, hiddenPosition.y, wi, 0), (float4)(weightPrev * scale + weightAlpha * learn, 0.0f, 0.0f, 0.0f));
+                write_imagef(weightsFront, (int4)(hiddenPosition.x, hiddenPosition.y, wi, 0), (float4)(fmin(1.0f, fmax(0.0f, weightPrev + weightAlpha * learn)), 0.0f, 0.0f, 0.0f));
             }
         }
 }

resources/neoKernelsCommon.cl

@@ -17,19 +17,19 @@ constant sampler_t normalizedClampedNearestSampler = CLK_NORMALIZED_COORDS_TRUE
     CLK_FILTER_NEAREST;
 
 constant sampler_t normalizedClampedToEdgeNearestSampler = CLK_NORMALIZED_COORDS_TRUE |
-    CLK_ADDRESS_CLAMP_TO_EDGE |
+    CLK_ADDRESS_CLAMP |
     CLK_FILTER_NEAREST;
 
 constant sampler_t unnormalizedClampedNearestSampler = CLK_NORMALIZED_COORDS_FALSE |
     CLK_ADDRESS_CLAMP |
     CLK_FILTER_NEAREST;
 
 constant sampler_t defaultNormalizedSampler = CLK_NORMALIZED_COORDS_TRUE |
-    CLK_ADDRESS_CLAMP_TO_EDGE |
+    CLK_ADDRESS_CLAMP |
     CLK_FILTER_NEAREST;
 
 constant sampler_t defaultUnnormalizedSampler = CLK_NORMALIZED_COORDS_FALSE |
-    CLK_ADDRESS_CLAMP_TO_EDGE |
+    CLK_ADDRESS_CLAMP |
     CLK_FILTER_NEAREST;
 
 // ----------------------------------------- Common -----------------------------------------
@@ -83,6 +83,10 @@ int2 project(int2 position, float2 toScalars) {
     return (int2)((position.x + 0.5f) * toScalars.x, (position.y + 0.5f) * toScalars.y);
 }
 
+int2 projectf(float2 position, float2 toScalars) {
+    return (int2)(position.x * toScalars.x, position.y * toScalars.y);
+}
+
 // Initialize a random uniform 2D image (X field)
 void kernel randomUniform2D(write_only image2d_t values, uint2 seed, float2 minMax) {
     uint2 seedValue = seed + (uint2)(get_global_id(0) * 29 + 12, get_global_id(1) * 16 + 23) * 36;

source/neo/AgentSwarm.h

@@ -45,11 +45,11 @@ namespace ogmaneo {
             \brief Initialize defaults
             */
             AgentLayerDesc()
-                : _radius(12), _qAlpha(0.1f),
-                _qGamma(0.99f), _qLambda(0.98f),
-                _epsilon(0.08f),
+                : _radius(12), _qAlpha(0.3f),
+                _qGamma(0.994f), _qLambda(0.99f),
+                _epsilon(0.04f),
                 _chunkSize({ 8, 8 }),
-                _chunkGamma(0.5f)
+                _chunkGamma(0.0f)
             {}
 
             //!@{

source/neo/Predictor.h

@@ -38,7 +38,7 @@ namespace ogmaneo {
             \brief Initialize defaults
             */
             PredLayerDesc()
-                : _radius(10), _alpha(0.1f), _beta(0.2f)
+                : _radius(8), _alpha(0.1f), _beta(0.3f)
             {}
 
             //!@{

source/neo/PredictorLayer.h

@@ -41,7 +41,7 @@ namespace ogmaneo {
             */
             VisibleLayerDesc()
                 : _size({ 16, 16 }),
-                _radius(10),
+                _radius(8),
                 _alpha(0.1f)
             {}
 

source/neo/SparseFeaturesChunk.cpp

@@ -88,7 +88,7 @@ SparseFeaturesChunk::SparseFeaturesChunk(ComputeSystem &cs, ComputeProgram &sfcP
 
     _hiddenSummationTemp = createDoubleBuffer2D(cs, _hiddenSize, CL_R, CL_FLOAT);
 
-    cs.getQueue().enqueueFillImage(_hiddenStates[_back], cl_float4{ 0.0f, 0.0f, 0.0f, 0.0f }, zeroOrigin, hiddenRegion);
+    cs.getQueue().enqueueFillImage(_hiddenStates[_back], cl_float4{ 0.0f, 1.0f, 0.0f, 0.0f }, zeroOrigin, hiddenRegion);
     cs.getQueue().enqueueFillImage(_hiddenActivations[_back], cl_float4{ 0.0f, 0.0f, 0.0f, 0.0f }, zeroOrigin, hiddenRegion);
 
     // Create kernels
@@ -328,7 +328,7 @@ flatbuffers::Offset<schemas::SparseFeaturesChunkDesc> SparseFeaturesChunk::Spars
         visibleLayerDescs.push_back(layerDesc.save(builder, cs));
 
     return schemas::CreateSparseFeaturesChunkDesc(builder,
-        &hiddenSize, &chunkSize,
+        &hiddenSize, &chunkSize, _numSamples,
         &initWeightRange, builder.CreateVectorOfStructs(visibleLayerDescs));
 }
 
@@ -345,6 +345,8 @@ void SparseFeaturesChunk::load(const schemas::SparseFeatures* fbSparseFeatures,
     _hiddenSize = cl_int2{ fbSparseFeaturesChunk->_hiddenSize()->x(), fbSparseFeaturesChunk->_hiddenSize()->y() };
     _chunkSize = cl_int2{ fbSparseFeaturesChunk->_chunkSize()->x(), fbSparseFeaturesChunk->_chunkSize()->y() };
 
+    _numSamples = fbSparseFeaturesChunk->_numSamples();
+
     ogmaneo::load(_hiddenStates, fbSparseFeaturesChunk->_hiddenStates(), cs);
     ogmaneo::load(_hiddenActivations, fbSparseFeaturesChunk->_hiddenActivations(), cs);
     ogmaneo::load(_chunkWinners, fbSparseFeaturesChunk->_chunkWinners(), cs);
@@ -375,7 +377,7 @@ flatbuffers::Offset<schemas::SparseFeatures> SparseFeaturesChunk::save(flatbuffe
         ogmaneo::save(_hiddenStates, builder, cs),
         ogmaneo::save(_hiddenActivations, builder, cs),
         ogmaneo::save(_chunkWinners, builder, cs),
-        &hiddenSize, &chunkSize,
+        &hiddenSize, &chunkSize, _numSamples,
         ogmaneo::save(_hiddenSummationTemp, builder, cs),
         builder.CreateVectorOfStructs(visibleLayerDescs),
         builder.CreateVector(visibleLayers));

source/neo/SparseFeaturesChunk.fbs

@@ -30,6 +30,7 @@ table VisibleChunkLayer {
 table SparseFeaturesChunkDesc {
 	_hiddenSize:int2;
     _chunkSize:int2;
+	_numSamples:int;
 	_initWeightRange:float2;
 	_visibleLayerDescs:[VisibleChunkLayerDesc];
 }
@@ -40,6 +41,7 @@ table SparseFeaturesChunk {
     _chunkWinners:DoubleBuffer2D;
     _hiddenSize:int2;
     _chunkSize:int2;
+	_numSamples:int;
     _hiddenSummationTemp:DoubleBuffer2D;
     _visibleLayerDescs:[VisibleChunkLayerDesc];
     _visibleLayers:[VisibleChunkLayer];

source/neo/SparseFeaturesChunk.h

@@ -53,7 +53,7 @@ namespace ogmaneo {
             */
             VisibleLayerDesc()
                 : _size({ 8, 8 }), _radius(8), _ignoreMiddle(false),
-                _weightAlpha(0.01f), _lambda(0.9f)
+                _weightAlpha(0.04f), _lambda(0.97f)
             {}
 
             //!@{
@@ -130,7 +130,7 @@ namespace ogmaneo {
             SparseFeaturesChunkDesc()
                 : _hiddenSize({ 36, 36 }),
                 _chunkSize({ 6, 6 }),
-                _numSamples(1),
+                _numSamples(2),
                 _initWeightRange({ -0.01f, 0.01f }),
                 _rng()
             {

source/neo/SparseFeaturesSTDP.h

@@ -52,7 +52,7 @@ namespace ogmaneo {
             \brief Initialize defaults
             */
             VisibleLayerDesc()
-                : _size({ 8, 8 }), _radius(10), _ignoreMiddle(false),
+                : _size({ 8, 8 }), _radius(8), _ignoreMiddle(false),
                 _weightAlpha(0.01f), _lambda(0.9f)
             {}
 
@@ -125,8 +125,8 @@ namespace ogmaneo {
             SparseFeaturesSTDPDesc()
                 : _hiddenSize({ 16, 16 }),
                 _inhibitionRadius(6),
-                _biasAlpha(0.01f), _activeRatio(0.01f), _gamma(0.96f),
-                _initWeightRange({ -0.01f, 0.01f }),
+                _biasAlpha(0.001f), _activeRatio(0.01f), _gamma(0.96f),
+                _initWeightRange({ 0.0f, 0.05f }),
                 _rng()
             {
                 _name = "STDP";