Neural Networks: same output?

Question

Cy137 5 Newbie Poster

14 Years Ago

I keep receiving the same output for each record when I run it on the test data. I've tried numerous things...any suggestions? I've worked on this for almost twelve hours straight, not counting previous work. I'm hoping one of you guys/gals who know whats going on can point me in the right direction.

#include <iostream>
#include <fstream>
#include <string>
#include <sstream>
#include <vector>
#include <time.h>
#include <stdlib.h>
#include <stdio.h>
#include <math.h>
#include <algorithm>
#include <functional>
#include <ctime>
#include <cstdlib>

/*
* Backpropagating Multilayered Feedforward Neural Network
* With code assistance by Tejpal Chhabra
* Written by:
* Zachary Bush(s0907933)
* Jesse Bordoe(s0907715)
*/
class CBackProp{

	//output of each neuron
	double **out;

	//3-D array to store weights for each neuron
	double ***weight;

	//delta error value for each neuron
	double **delta;

	//no of layers in net including input layer
	int numl;

	//array of numl elements to store size of each layer
	int *lsize;

	//learning rate
	double beta;

	//momentum
	double alpha;

	//storage for weight-change made in previous epoch
	double ***prevDwt;

	//sigmoid function
	double sigmoid(double in);

public:

	~CBackProp();

	//initializes and allocates memory
	CBackProp(int nl,int *sz,double b,double a);

	//backpropogates error for one set of input
	void bpgt(double *in,double *tgt);

	//feed forwards activations for one set of inputs
	void ffwd(double *in);

	//returns mean square error of the net
	double mse(double *tgt) const;

	//returns i'th output of the net
	double Out(int i) const;

	//get weight values
	double getWeight(int i, int j, int k);


};

//get weight values
double CBackProp::getWeight(int i, int j, int k)
{
	return weight[i][j][k];
}

//	initializes and allocates memory on heap
CBackProp::CBackProp(int nl,int *sz,double b,double a):beta(b),alpha(a)
{

	//	set no of layers and their sizes
	numl=nl;
	lsize=new int[numl];

	for(int i=0;i<numl;i++){
		lsize[i]=sz[i];
	}

	//	allocate memory for output of each neuron
	out = new double*[numl];

	for(int i=0;i<numl;i++){
		out[i]=new double[lsize[i]];
	}

	//	allocate memory for delta
	delta = new double*[numl];

	for(int i=1;i<numl;i++){
		delta[i]=new double[lsize[i]];
	}

	//	allocate memory for weights
	weight = new double**[numl];

	for(int i=1;i<numl;i++){
		weight[i]=new double*[lsize[i]];
	}
	for(int i=1;i<numl;i++){
		for(int j=0;j<lsize[i];j++){
			weight[i][j]=new double[lsize[i-1]+1];
		}
	}

	//	allocate memory for previous weights
	prevDwt = new double**[numl];

	for(int i=1;i<numl;i++){
		prevDwt[i]=new double*[lsize[i]];

	}
	for(int i=1;i<numl;i++){
		for(int j=0;j<lsize[i];j++){
			prevDwt[i][j]=new double[lsize[i-1]+1];
		}
	}

	//	seed and assign random weights
	srand((unsigned)(time(NULL)));
	for(int i=1;i<numl;i++)
		for(int j=0;j<lsize[i];j++)
			for(int k=0;k<lsize[i-1]+1;k++)
				weight[i][j][k]=(double)(rand())/(RAND_MAX/2) - 1;//32767

	//	initialize previous weights to 0 for first iteration
	for(int i=1;i<numl;i++)
		for(int j=0;j<lsize[i];j++)
			for(int k=0;k<lsize[i-1]+1;k++)
				prevDwt[i][j][k]=(double)0.0;


}



CBackProp::~CBackProp()
{
	//	free out
	for(int i=0;i<numl;i++)
		delete[] out[i];
	delete[] out;

	//	free delta
	for(int i=1;i<numl;i++)
		delete[] delta[i];
	delete[] delta;

	//	free weight
	for(int i=1;i<numl;i++)
		for(int j=0;j<lsize[i];j++)
			delete[] weight[i][j];
	for(int i=1;i<numl;i++)
		delete[] weight[i];
	delete[] weight;

	//	free prevDwt
	for(int i=1;i<numl;i++)
		for(int j=0;j<lsize[i];j++)
			delete[] prevDwt[i][j];
	for(int i=1;i<numl;i++)
		delete[] prevDwt[i];
	delete[] prevDwt;

	//	free layer info
	delete[] lsize;
}

//	sigmoid function
double CBackProp::sigmoid(double in)
{
	return (double)(1/(1+exp(-in)));
}

//	mean square error
double CBackProp::mse(double *tgt) const
{
	double mse=0;
	for(int i=0;i<lsize[numl-1];i++){
		mse+=(tgt[i]-out[numl-1][i])*(tgt[i]-out[numl-1][i]);
	}
	return mse/2;
}


//	returns i'th output of the net
double CBackProp::Out(int i) const
{
	return out[numl-1][i];
}

// feed forward one set of input
void CBackProp::ffwd(double *in)
{
	double sum;

	//	assign content to input layer
	for(int i=0;i<lsize[0];i++)
		out[0][i]=in[i];  // output_from_neuron(i,j) Jth neuron in Ith Layer

	//	assign output(activation) value
	//	to each neuron usng sigmoid func
	for(int i=1;i<numl;i++){				// For each layer
		for(int j=0;j<lsize[i];j++){		// For each neuron in current layer
			sum=0.0;
			for(int k=0;k<lsize[i-1];k++){		// For input from each neuron in preceeding layer
				sum+= out[i-1][k]*weight[i][j][k];	// Apply weight to inputs and add to sum
			}
			sum+=weight[i][j][lsize[i-1]];		// Apply bias
			out[i][j]=sigmoid(sum);				// Apply sigmoid function
		}
	}
}


//	backpropogate errors from output
//	layer uptill the first hidden layer
void CBackProp::bpgt(double *in,double *tgt)
{
	double sum;

	//	update output values for each neuron
	ffwd(in);

	//	find delta for output layer
	for(int i=0;i<lsize[numl-1];i++){
		delta[numl-1][i]=out[numl-1][i]*
			(1-out[numl-1][i])*(tgt[i]-out[numl-1][i]);
	}

	//	find delta for hidden layers
	for(int i=numl-2;i>0;i--){
		for(int j=0;j<lsize[i];j++){
			sum=0.0;
			for(int k=0;k<lsize[i+1];k++){
				sum+=delta[i+1][k]*weight[i+1][k][j];
			}
			delta[i][j]=out[i][j]*(1-out[i][j])*sum;
		}
	}

	//	apply momentum ( does nothing if alpha=0 )
	for(int i=1;i<numl;i++){
		for(int j=0;j<lsize[i];j++){
			for(int k=0;k<lsize[i-1];k++){
				weight[i][j][k]+=alpha*prevDwt[i][j][k];
			}
			weight[i][j][lsize[i-1]]+=alpha*prevDwt[i][j][lsize[i-1]];
		}
	}

	//	adjust weights usng steepest descent
	for(int i=1;i<numl;i++){
		for(int j=0;j<lsize[i];j++){
			for(int k=0;k<lsize[i-1];k++){
				prevDwt[i][j][k]=beta*delta[i][j]*out[i-1][k];
				weight[i][j][k]+=prevDwt[i][j][k];
			}
			prevDwt[i][j][lsize[i-1]]=beta*delta[i][j];
			weight[i][j][lsize[i-1]]+=prevDwt[i][j][lsize[i-1]];
		}
	}
}



using namespace std;

int stringToInt(string convert);

int main(){
	string answers;
	string dataset;
	string directory;
	string answersDirectory;
	string datasetDirectory;
	size_t found;
	string currentFile=__FILE__;
	string stringHolder;
	int intHolder;
	vector<vector<double> > input;
	vector<vector<double> > testData;
	int records;
	int features;
	int y=0;
	int z=0;
	int numLayers=3;
	int layerSize[3]={3,6,1}; //prev 3
	double beta = 0.3;
	double alpha = 0.1;
	double thresh =  0.00001;
	double result = 0;
	int num_iter = 1000; //prev 1000
	int i=0;
	int correct=0;
	int recordCount=0;
	double accuracy;
	vector<int> randomize;

	//randomize
	srand(time(NULL));

	answers="answers.dat";
	dataset="dataset.dat";

	found=currentFile.find_last_of("\\");
	directory=currentFile.substr(0,found);
	directory+="\\data\\";

	answersDirectory=directory + answers;
	datasetDirectory=directory + dataset;

	ifstream openAnswers;
	ifstream openDataset;

	openAnswers.open(answersDirectory.c_str());
	openDataset.open(datasetDirectory.c_str());

	if(openAnswers.is_open()&&openDataset.is_open())
	{
		getline(openDataset,stringHolder,' ');
		intHolder=stringToInt(stringHolder);
		records=intHolder;

		getline(openDataset,stringHolder,' ');
		intHolder=stringToInt(stringHolder);
		features=intHolder;

		layerSize[0]=records;

		input.resize(records*2);
		testData.resize(records*2);
		for(int x=0; x < (records); x++)
		{
			input[x].resize((features+1)*2);
			testData[x].resize((features+1)*2);
			cout << "Resizing vector to fit records." << endl;
		}

		getline(openAnswers,stringHolder);		//To remove 'records' from the top of answers.dat

		while(!openAnswers.eof() && !openDataset.eof() && recordCount<records)
		{
			if(openDataset.peek()=='\n')
			{
				getline(openDataset,stringHolder);
				getline(openAnswers,stringHolder);
				if(stringHolder=="ad")
					input[y][z]=1;
				else
					input[y][z]=0;
				y++;
				z=0;
				cout << "new record added, finished record: " << recordCount <<endl;
				recordCount++;

			}
			else
			{
				getline(openDataset,stringHolder,' ');
				intHolder=stringToInt(stringHolder);
				input[y][z]=intHolder;
				z++;
				cout << "filling vector with values. record #: " << recordCount << endl;
			}
		}



		cout << "Files opened successfully" << endl;
	}
	else
		cout << "Files failed opening."  << endl;

	for(int i=0;i<records;i++)
		randomize.push_back(i);
	random_shuffle(randomize.begin(),randomize.end());

	CBackProp *bp = new CBackProp(numLayers, layerSize, beta, alpha);
	cout<< endl <<  "Now training the network...." << endl;


	for (int i=0; i<num_iter ; i++)
	{
		double* ptr;

		ptr=(double*) &input[i%records];

		bp->bpgt(ptr, &input[i%records][features]);

		if( bp->mse(&input[i%records][features]) < thresh) {
			cout << endl << "Network Trained. threshold value achieved in " << i << "iterations." << endl;
			cout << "MSE:  " << bp->mse(&input[i%records][features])
				<<  endl <<  endl;
			break;
		}
		if ( i%(num_iter/10) == 0 )
			cout<<  endl <<  "MSE:  " << bp->mse(&input[i%records][features])
			<< "... Training..." << endl;


	}

	if ( i == num_iter )
		cout << endl << i << " iterations completed..."	<< "MSE: " << bp->mse(&input[(i-1)%records][features]) << endl;

	cout<< "Now using the trained network to make predctions on test input...." << endl << endl;
	double* ptr2;
	//ptr2=(double*) &testData[i];
	/*recordCount=0;
	for(int i=0;i<records;i++)
		randomize.push_back(i);
	random_shuffle(randomize.begin(),randomize.end());

	while(!randomize.empty())
	{
		i=randomize.back();
		randomize.pop_back();
		ptr2=(double*) &testData[i];
		bp->ffwd(ptr2);
		cout << recordCount << " " << bp->Out(0) << endl;
		recordCount++;
	}
	*/
	for(int i=0;i<records;i++)
		randomize.push_back(i);
	random_shuffle(randomize.begin(),randomize.end());
	
	ofstream best;
	best.open("data//best.nn");


	if(best.is_open())
	{
		ifstream check;
		check.open(answersDirectory.c_str());
		if(check.is_open())
		{
			cout << "Files opened successfully" << endl;
			getline(check,stringHolder);		//To remove 'records' from the top of answers.dat
			/*while(!check.eof() && recordCount<records)
			{
			getline(check,stringHolder);

			if(stringHolder=="ad" && bp->Out(0) >.5)
			correct++;
			else if(stringHolder=="nonad" && bp->Out(0) <.5)
			correct++;
			recordCount++;
			}
			accuracy=correct/recordCount;
			best << accuracy << endl; 
			cout << "Files opened successfully" << endl;*/
			getline(check,stringHolder); //To remove 'records' from the top of answers.dat
			for (int i = 0 ; i < records ; i++ )
			{
				ptr2=(double*) &testData[i];
				bp->ffwd(ptr2);
				result=bp->Out(0);
				cout << i << "  "  << result << endl;
				getline(check,stringHolder);
				if(stringHolder=="ad" && result >.5)
					correct++;
				else if(stringHolder=="nonad" && result <.5)
					correct++;
			}
			accuracy=correct/records;
			best << accuracy << endl;


			best << numLayers-2 << endl;
			best << "I " << records << " H " << layerSize[1] << endl;
			for(int k=0; k<layerSize[1]; k++)
			{
				for(int j=0;j<records;j++)
					//best << bp->getWeight(1,j,k) << " ";
					best << bp->getWeight(1,k,j) << " ";
			}
			best << endl << "H " << layerSize[1] << " O 1" << endl;
			for(int j=0; j<layerSize[2]; j++)

				for(int k=0;k<layerSize[1]; k++)
					best << bp->getWeight(2,j,k) << " ";
		}
		else
			cout << "answers failed to open" << endl;
		best.close();
	}
	else
		cout << "best failed to write. " << endl;

	return 0;
}

int stringToInt(string convert)
{
	int returnInt;
	istringstream integer(convert);
	integer >> returnInt;
	return returnInt;
}

Small names file:

35
ad
nonad
ad
ad
nonad
ad
nonad
ad
ad
nonad
ad
nonad
nonad
nonad
nonad
ad
ad
ad
nonad
nonad
ad
nonad
ad
nonad
nonad
ad
nonad
nonad
ad
nonad
nonad
ad
ad
nonad
nonad

small dataset:

35 50
1 0 0 1 0 1 0 0 1 0 0 1 1 1 1 0 0 1 0 1 1 1 0 1 1 0 1 0 0 1 1 1 0 0 1 0 0 0 1 1 0 1 0 1 1 1 0 0 0 0 
0 0 0 0 1 1 1 1 0 0 0 1 1 0 1 1 0 1 1 1 0 1 1 1 0 1 0 1 1 1 1 0 1 1 1 0 0 1 0 1 1 0 1 0 1 0 1 0 1 1 
1 0 0 0 1 1 1 0 1 1 0 1 1 0 1 1 1 0 1 1 0 1 0 1 0 1 1 0 0 1 1 0 0 1 0 0 1 0 1 0 0 1 0 0 1 1 1 1 0 1 
0 1 0 1 0 0 1 1 0 0 1 1 1 1 0 0 0 1 0 0 0 0 0 0 1 0 0 1 1 1 0 0 0 0 0 1 1 1 0 1 1 1 1 1 1 0 1 1 0 1 
1 1 1 0 1 1 1 0 0 1 0 0 0 0 1 1 1 1 1 1 0 0 1 0 0 0 0 0 1 0 1 0 1 0 0 1 0 0 0 0 0 1 1 0 1 0 1 1 1 1 
0 0 1 0 0 0 1 0 1 0 0 1 1 0 0 1 1 0 0 0 1 1 0 1 0 1 1 1 1 0 1 1 1 1 1 0 0 0 1 1 1 0 0 1 1 0 0 0 1 1 
0 1 0 0 1 0 1 0 0 0 0 0 0 1 0 1 0 0 0 0 1 0 0 1 0 1 1 1 1 0 1 0 1 1 0 0 0 1 1 1 0 0 0 1 0 1 0 1 0 1 
0 1 0 1 0 0 0 1 0 0 0 1 0 0 0 0 0 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 0 1 0 1 0 0 1 0 0 0 0 0 0 0 1 0 0 
0 1 1 0 1 0 1 0 0 1 1 0 1 0 0 0 0 1 1 1 0 0 0 1 1 1 0 1 0 0 0 0 1 1 0 1 1 1 1 0 1 0 1 1 0 1 1 0 0 0 
0 1 0 1 0 0 1 0 1 0 1 0 1 1 1 0 1 1 0 0 1 0 1 0 0 0 0 0 1 1 0 0 0 1 0 1 1 1 0 1 0 1 1 0 1 0 1 0 0 0 
0 0 1 1 1 0 0 0 0 0 1 1 1 0 1 0 1 1 1 0 1 1 0 1 0 1 1 0 1 0 1 0 1 1 1 1 1 0 0 0 1 1 1 0 1 1 1 0 0 1 
1 1 0 0 0 0 0 1 1 1 0 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 1 1 1 0 0 1 0 1 1 1 0 1 0 0 0 0 1 1 1 0 1 1 1 0 
0 1 0 1 1 1 0 1 1 1 1 1 0 0 0 1 0 1 0 1 0 0 1 0 1 1 1 1 1 1 1 1 1 1 0 0 0 1 1 1 0 1 0 0 0 0 0 0 0 0 
1 1 0 1 0 1 1 0 0 1 0 0 0 1 0 1 0 1 0 1 0 1 1 1 1 1 0 1 1 0 0 1 1 0 1 0 1 0 0 1 0 0 0 1 0 0 1 1 1 0 
1 0 0 0 1 1 0 0 0 0 0 0 1 1 0 0 1 1 0 1 0 1 1 1 0 1 1 1 1 0 0 1 1 1 1 1 0 0 0 0 1 0 0 1 0 1 1 1 0 1 
0 1 0 0 0 1 0 0 0 0 0 1 1 0 0 0 0 1 0 0 1 0 0 1 1 0 1 0 1 0 1 1 0 0 1 1 0 1 1 0 0 1 1 1 0 0 1 0 1 1 
0 1 1 0 0 1 1 0 1 0 0 0 0 1 1 1 0 1 0 1 1 1 0 0 1 1 0 1 0 1 0 0 1 0 0 0 1 0 1 1 1 1 1 1 1 0 1 1 0 0 
1 0 1 1 0 1 0 1 1 0 0 0 1 1 0 1 1 1 1 1 0 1 0 1 0 1 0 0 0 0 1 1 0 1 0 0 0 0 1 0 1 0 0 1 0 1 1 0 0 0 
1 0 1 1 0 0 0 1 0 1 0 1 0 0 1 1 0 0 1 0 1 0 1 0 0 1 0 1 0 0 1 0 1 1 0 0 1 1 1 0 0 1 0 0 1 1 0 0 1 0 
1 1 0 0 1 0 1 0 0 1 1 1 1 1 0 1 0 0 1 0 1 0 0 1 0 1 0 1 0 1 1 0 0 1 1 1 0 0 1 1 0 0 1 0 0 1 0 1 1 0 
1 0 0 1 1 0 0 1 1 0 0 0 1 0 1 0 1 1 1 0 1 1 1 1 1 1 1 0 0 0 0 0 0 1 1 1 0 0 0 1 1 1 0 0 0 1 1 1 1 1 
1 1 0 1 1 1 0 0 0 1 1 0 1 1 0 0 0 1 1 0 1 1 1 0 0 0 0 1 0 1 0 1 1 0 1 1 0 0 0 1 0 0 0 1 1 0 1 1 0 0 
1 0 0 0 0 0 0 0 0 1 1 0 0 0 1 0 1 1 1 1 0 1 0 0 0 0 1 1 1 1 0 1 1 1 1 0 0 1 1 0 1 0 0 1 1 1 0 1 0 0 
1 1 1 1 0 1 1 1 1 0 0 0 0 0 1 0 1 1 0 1 0 0 1 0 1 1 1 0 0 0 0 1 1 0 1 0 1 1 1 1 0 0 1 1 1 0 1 1 0 0 
0 0 0 0 0 1 1 0 0 0 1 1 1 0 1 1 0 1 0 0 1 1 0 1 1 1 0 1 0 1 1 1 0 1 0 1 0 0 1 0 0 0 1 0 1 0 0 0 1 1 
1 0 0 1 1 0 1 1 0 0 0 1 0 1 0 0 0 0 1 1 1 1 0 0 0 0 1 1 0 1 0 0 0 0 1 0 0 0 0 1 0 0 0 1 0 1 1 0 0 0 
1 0 1 1 1 0 1 0 0 1 1 0 0 0 1 0 0 0 0 0 1 1 0 1 0 0 1 0 1 0 0 0 1 1 0 0 0 0 1 0 0 1 1 1 0 0 0 0 1 1 
1 0 1 0 1 0 1 0 0 0 1 0 1 1 1 0 0 0 1 1 1 1 0 1 0 1 1 1 1 0 1 1 0 0 0 1 1 1 0 1 1 1 1 1 0 0 1 1 0 0 
0 1 1 1 0 0 1 0 0 0 0 1 1 0 1 0 1 1 1 0 1 1 1 1 1 0 1 0 1 1 1 0 1 0 0 0 1 0 1 1 1 0 1 1 1 0 1 1 1 0 
0 0 0 1 1 0 1 0 1 1 0 1 0 1 1 1 0 1 1 1 0 0 0 0 0 1 0 0 0 1 0 1 0 1 1 1 0 1 1 1 1 0 0 0 1 1 0 1 1 1 
0 1 1 1 0 1 1 1 1 1 0 1 1 0 0 0 1 1 0 1 1 1 1 0 0 1 0 0 0 0 0 1 0 1 0 1 1 0 0 1 1 1 1 1 0 1 1 0 0 1 
1 1 1 1 0 0 1 0 1 1 1 0 0 0 1 1 0 1 1 0 1 1 0 1 0 0 0 0 0 0 0 0 0 0 1 1 0 1 1 1 1 0 1 1 0 1 1 0 1 0 
1 0 0 1 1 0 0 1 1 0 0 1 1 1 1 1 1 1 1 1 1 0 1 1 0 1 1 1 1 1 0 0 1 0 0 0 1 0 1 1 1 1 1 1 0 0 0 0 0 1 
1 0 0 0 0 1 1 1 1 1 1 0 1 1 0 1 0 0 1 0 0 1 1 1 1 1 1 0 0 0 0 1 0 0 0 1 0 0 1 0 1 1 1 0 0 0 0 1 1 1 
1 1 0 0 1 1 0 1 0 0 1 1 0 1 1 0 0 1 0 0 0 0 0 1 0 1 1 0 0 1 0 1 1 0 0 1 1 0 0 0 1 1 0 0 1 1 0 1 1 1

I have the large datasets too, but they're on the scale of 2000 records and 1500 features...so I don't want to fill up the forums.

Thanks in advance.

c++

Edited 14 Years Ago by Cy137 because: n/a

2 Contributors
1 Reply
184 Views
6 Hours Discussion Span
Latest Post 14 Years Ago Latest Post by mrnutty

Reply to this topic

Be a part of the DaniWeb community

We're a friendly, industry-focused community of developers, IT pros, digital marketers, and technology enthusiasts meeting, networking, learning, and sharing knowledge.

mrnutty 761 Senior Poster · Answer 1 · 2009-12-13T23:27:47+00:00

>>I keep receiving the same output for each record when I run it on the test data. I've tried numerous things...any suggestions?

Yes, learn how to use the debugger! You can't just dump "huge" code on us and expect us to fix it for you. If you "step-by" your code, you will find your problem fast. It will help you in the long run if you know how
to use a debugger.