Dear All,
I have java listener which keep reading for incoming data to the particular port. Most of the times it works fine only at times it gets too many open files error. Below is my codes what could be wrong? Thank you.

public static void main(String[] args) {
 
 try 
 {
   final ServerSocket serverSocketConn = new ServerSocket(8888);
 
 
   while (true) 
   {
    try 
    {
       Socket socketConn1 = serverSocketConn.accept();
       new Thread(new ConnectionHandler(socketConn1)).start(); 
    }
   catch(Exception e)
   {
      System.out.println(e.toString());
    }
  }
 
 
  } 
  catch (Exception e) 
  {
  System.out.println(e.toString());
  //System.exit(0); 
  }
}
 
 
}
class ConnectionHandler implements Runnable 
{
   private Socket receivedSocketConn1;
   DateFormat dateFormat = new SimpleDateFormat("dd/MM/yyyy HH:mm:ss");
   DateFormat formatter = new SimpleDateFormat("EEE, dd MMM yyyy"); 
   DateFormat inDf=new SimpleDateFormat("ddMMyyHHmmss"); 
   DateFormat outDf=new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); 
 
   ConnectionHandler(Socket receivedSocketConn1) 
   {
   this.receivedSocketConn1=receivedSocketConn1;
   }
 
//@Override
public void run() 
{ 
      Connection dbconn = null;
      BufferedWriter w = null;
      BufferedReader r = null;
 try 
 {
 
 
         PrintStream out = System.out; 
         BufferedWriter fout = null;
         w = new BufferedWriter(new OutputStreamWriter(receivedSocketConn1.getOutputStream()));
         r = new BufferedReader(new InputStreamReader(receivedSocketConn1.getInputStream()));
 
         int m = 0, count=0;
         String line="";
         String n="";
         w.write("$PA\n");
         w.flush();
             while ((m=r.read()) != -1) 
             {
                  Date dateIn = new Date();
                  n = n + (char) m;
                  int i = n.indexOf("GET");
                  if(i != -1) 
                  { 
                     break;
                  }
 
 
                 if (m==35)
                 {
 
 
                   String ori = n;
                   String noCheckSum = n.substring(0,(n.length()-4));
                   int addExist = n.indexOf("@");
 
                   String[] slave = null;
                   if(addExist!=-1)
                   {
                     slave = noCheckSum.split("@");
                     n = slave[0];
                   }
                   else
                   {
                     n = noCheckSum;
                   }
 
                   String[] result = n.split(",");
                   Statement stmt = null;
 
                   w.write("$PA\n");
                   w.flush();
 
                   int count1 = 0;
 
                   Date date = Calendar.getInstance().getTime(); 
                   String today = formatter.format(date); 
                   String filename= "MyDataFile"today".txt"; 
                   boolean append = true; 
 
 
                   FileWriter fw = null;
                   try
                   {
                    fw = new FileWriter(filename,append); 
                    fw.write(ori+" "dateFormat.format(dateIn)"\n");//appends the string to the file 
                    Date dateOut = new Date();
                    fw.write("$PA"+" "dateFormat.format(dateOut)"\n");//appends the string to the file 
                   }
                   catch (IOException ex) 
                   { 
                   System.out.println("MyError:IOException has been caught in in the file operation");
                   ex.printStackTrace();
                   } 
                  finally
                  {
                    try 
                    {
                     if ( fw != null ) 
                     {
                       fw.close();
                     }
                     else 
                     {
                       System.out.println("MyError:fw is null in finally close");
                     }
                  }
                  catch(IOException ex){
                      System.out.println("MyError:IOException has been caught in fw is null in finally close");
                      ex.printStackTrace();
                  }
              }
              try
              {
 
 
               String deviceID=result[3].trim(); 
               String dateTime=result[4].trim(); 
               String[] result2 = result[10].split("'"); 
               String gpsDate = result2[1].trim().substring(0,6); 
               String gpsTime = result2[1].trim().substring(6,12); 
               String gpsLat = result2[1].trim().substring(13,20);
               String latitude = result2[1].trim().substring(13,20).substring(0,2)+"."+result2[1].trim().substring(13,20).substring(2,7);
               String gpsLong = result2[1].trim().substring(21,29);
               String longitude = result2[1].trim().substring(21,29).substring(0,3)+"."+result2[1].trim().substring(21,29).substring(3,8);
               String speed = result2[1].trim().substring(30,33); 
               String course = result2[1].trim().substring(33,36);
               String dateTimer = null;
               try
               { 
                   Date inDate=null; 
                   inDf.setTimeZone(TimeZone.getTimeZone("UTC")); 
                   inDate=inDf.parse(dateTime); 
                   outDf.setTimeZone(TimeZone.getTimeZone("Asia/Kuala_Lumpur")); 
                   dateTimer=outDf.format(inDate); 
               }
                  catch(ParseException ex)
               { 
                   System.out.println("MyError:Parse Error has been caught for date parse close");
                   ex.printStackTrace();
               } 
 
               dbconn = DriverManager.getConnection("jdbc:mysql://192.168.1.155:3306/db1?"+"user=db1&password=test1");
               stmt = dbconn.createStatement();
 
               String selectQuery2 = "Select * from tripData Where deviceID='"+ deviceID +"' and dateTimer>'"dateTimer"' Order By dateTimer Desc Limit 1"; 
               ResultSet rs2 = stmt.executeQuery(selectQuery2);
 
 
               String updateQuery = "";
               if(rs2.next())
               {
                  String previousLatitude="";
                  String previousLongitude="";
                  String previousSpeed="";
 
                  previousLatitude = rs2.getString("latitude");
                  previousLongitude = rs2.getString("longitude");
                  previousSpeed = rs2.getString("speed");
 
                  updateQuery = "UPDATE device SET " + 
                  "latitude='" + previousLatitude + 
                   "',longitude='" + previousLongitude + 
                   "',speed='" + previousSpeed + 
                   "' WHERE serialNumber='" + deviceID + "'";
               }
               else
               {
 
 
                  updateQuery = "UPDATE device SET " + 
                   "latitude='" + latitude + 
                   "',longitude='" + longitude + 
                   "',speed='" + speed + 
                   "',course='" + course +
                   "',dateTimer='" + dateTimer +
                   "' WHERE serialNumber='" + deviceID + "'";
                } 
                count = stmt.executeUpdate(updateQuery); 
 
 
                String insertQuery = "INSERT INTO tripData" + 
                 "(latitude,longitude,speed,course,dateTimer,deviceID)" + " VALUES ('" + 
                 latitude + "','" + longitude + "','" + speed + "','" + course + "','" + dateTimer + "','" + deviceID + "' )";
                count = stmt.executeUpdate(insertQuery);
 
 
               if(addExist!=-1)
               {
                 for(int iSlave=1; iSlave<slave.length ; iSlave++)
                 {
                    String[] slaveDetails = slave[iSlave].split(",",-1);
                    String slaveEventType = slaveDetails[0];
                    String slaveGroup = slaveDetails[1];
                    String slaveUnitID = slaveDetails[2];
                    String slaveBattLevel = slaveDetails[3];
                    String slaveSwitchStat = slaveDetails[4];
                    String slaveTempHumid = slaveDetails[5];
                    String slaveTemp="",slaveHumid=""; 
                    if(slaveTempHumid.length()>0)
                     {
                      slaveHumid = slaveTempHumid.substring(5,7);
                      if(slaveTempHumid.charAt(0)=='P')
                      {
                        slaveTemp = "+"slaveTempHumid.substring(1,3)"."+slaveTempHumid.substring(3,5); 
                      }
                      else if(slaveTempHumid.charAt(0)=='M')
                      {
                        slaveTemp = "-"slaveTempHumid.substring(1,3)"."+slaveTempHumid.substring(3,5);
                       }
                     }
                    slaveBattLevel = slaveBattLevel.trim().substring(0,2)+"."+slaveBattLevel.trim().substring(2,3);
                    String insertQuery2 = "INSERT INTO tripDataSlave" + 
                     "(dateTimer,deviceID,slaveUnitID,slaveEventType,slaveGroup,slaveBattLevel,slaveSwitchStat,slaveTemp                     ,slaveHumidity)" + " VALUES ('" + dateTimer + "','" + deviceID + "','" + slaveUnitID + "','" +   slaveEventType + "','" + slaveGroup + "','" + slaveBattLevel + "','" + slaveSwitchStat + "','" + slaveTemp + "','" +  slaveHumid + "')";
                    count = stmt.executeUpdate(insertQuery2);
                  }
               }
            }
           catch (SQLException ex) 
           { 
            System.out.println("MyError:Error : "+ex);
 
           } 
           finally
          {
           try 
           {
              if ( stmt != null ) 
              {
               stmt.close();
              }
              else 
              {
                System.out.println("MyError:stmt is null in finally close");
              }
            }
            catch(SQLException ex){
                  System.out.println("MyError:SQLException has been caught for stmt close");
                  ex.printStackTrace();
            }
            try 
            {
             if ( dbconn != null ) 
             {
               dbconn.close();
             }
              else 
             {
                System.out.println("MyError:dbconn is null in finally close");
              }
            }
            catch(SQLException ex){
                 System.out.println("MyError:SQLException has been caught for dbconn close");
                 ex.printStackTrace();
            }
 
 
      }
       n="";
     }
  }
 } 
 catch (IOException ex) 
 { 
 System.out.println("MyError:IOException has been caught in in the main first try");
 ex.printStackTrace();
 } 
 finally
 {
   try 
   {
    if ( w != null ) 
    {
      w.close();
    }
    else 
    {
     System.out.println("MyError:w is null in finally close");
    }
   }
    catch(IOException ex){
     System.out.println("MyError:IOException has been caught in w in finally close");
    ex.printStackTrace();
   }
 
  }
 }
}

What operating system are you running on? Do you have the entire stack trace?

This error normally comes up when you end up depleting your quota of the maximum acceptable file handles which can be open at a given time which in turn happens if you don't close file/socket streams which you have opened. By having a look at your code, it seems that you never close the InputStream retrieved from the client socket (the variable ' r ') which might be what is causing these errors.

A few more things:

  1. Split your logic into methods instead of pushing the entire thing is just a single method
  2. Instead of concatenating queries, use PreparedStatement which are immune to SQL injection. For repeated execution they are faster since once a statement is "prepared" the cost of re-compilation of query is no longer there.
  3. Spawning threads on demand is expensive; use thread pools. If you are using Java >= 5, look into ExecutorService .
  4. Spawning connections on demand is also expensive; use connection pooling if your profiling suggests that the application takes a lot of time in "preparing" database resources

Dear SOS,
My O.S is fedora linux. I do not know how to get the entire stack trace printed? Actually my program is running in a wrapper program. So normally anything I system.out.println will print in the wrapper log file. So do you think I should do the same for the stack trace? Another thing regarding the closing of the input stream I read that if w is close then automatically it take of the r too. I dont know how to split the logic into different function as there are all related to each other I find it quite difficult to split too. Yes I am using java 5. I will try to look into the executor service and also other pooling. I am sorry quite new into all this pooling etc.

What operating system are you running on? Do you have the entire stack trace?

This error normally comes up when you end up depleting your quota of the maximum acceptable file handles which can be open at a given time which in turn happens if you don't close file/socket streams which you have opened. By having a look at your code, it seems that you never close the InputStream retrieved from the client socket (the variable ' r ') which might be what is causing these errors.

A few more things:

  1. Split your logic into methods instead of pushing the entire thing is just a single method
  2. Instead of concatenating queries, use PreparedStatement which are immune to SQL injection. For repeated execution they are faster since once a statement is "prepared" the cost of re-compilation of query is no longer there.
  3. Spawning threads on demand is expensive; use thread pools. If you are using Java >= 5, look into ExecutorService .
  4. Spawning connections on demand is also expensive; use connection pooling if your profiling suggests that the application takes a lot of time in "preparing" database resources

My O.S is fedora linux. I do not know how to get the entire stack trace printed?

Just make sure that you don't catch any exceptions and gobble them (i.e. don't do anything). Either log that error using Log4J's FATAL logging level or at least do a e.printStackTrace() . Also, make sure that when you start this application on your *nix box, you don't redirect the error stream to /dev/null which might end up gobbling up all your stack traces which might have been printed.

If there was no exception stack trace, how did you know the error was 'too many open files'? Also, how are you starting the application on your *nix box? As a background process?

So normally anything I system.out.println will print in the wrapper log file. So do you think I should do the same for the stack trace?

Exception stack traces are printed out to the *error* stream instead of the standard output stream, so just make sure that your wrapper program captures both error and standard output streams (STDOUT, STDERR).

Another thing regarding the closing of the input stream I read that if w is close then automatically it take of the r too

I don't think so since nothing along those lines is written in the Javadocs for the Socket class. I'd rather play safe and close *all* open resources which were opened; this involves closing socket, input stream and output stream.

I dont know how to split the logic into different function as there are all related to each other I find it quite difficult to split too

If you haven't done any re-factoring before, of course you'd find it difficult. I'd suggest looking at your application as a collection of components, components which:
1) Parse the incoming user input
2) The database component which retrieves data using the user input
3) Render output to user

You can also wrap up all that resource closing code in a single utility method to avoid repetitive boiler plate code.

Follow the above steps (at least the closing all resources thing), load test your application (an automated script which hits your application continuously) and see if you again get the same problem. Ideally, the default open file descriptors allowed (which I guess is 4096) should be good enough for your case.

Edited 5 Years Ago by ~s.o.s~: n/a

Dear Sos,
Ok let me explain how I run this. I am using the the YAJSW wrapper from this site http://yajsw.sourceforge.net/. So yes it runs in the background.Below is a sample from the wrapper's log file. Each time I am actually wrting to a text file too. Is this the problem is it?

FINEST|22986/0|10-12-09 15:56:04|java.io.FileNotFoundException: MyDataFileThu, 09 Dec 2010.txt (Too many open files)
INFO|wrapper|10-12-09 15:56:04|Trigger found: Too many open files
INFO|wrapper|10-12-09 15:56:04|set state RUNNING->RESTART
INFO|wrapper|10-12-09 15:56:04|restarting 1 time
INFO|wrapper|10-12-09 15:56:04|set state RESTART->RESTART_STOP
INFO|wrapper|10-12-09 15:56:04|stopping process with pid/timeout 22986 45000
INFO|22986/0|10-12-09 15:56:04|Controller State: LOGGED_ON -> WAITING_CLOSED
INFO|22986/0|10-12-09 15:56:04|MyError:IOException has been caught in in the file operation
INFO|22986/0|10-12-09 15:56:04|MyError:fw is null in finally close
INFO|22986/0|10-12-09 15:56:04|wrapper manager received stop command
INFO|22986/0|10-12-09 15:56:04|MyError:Error : com.mysql.jdbc.exceptions.jdbc4.CommunicationsException: Communications link failure
INFO|22986/0|10-12-09 15:56:04|
INFO|22986/0|10-12-09 15:56:04|The last packet sent successfully to the server was 0 milliseconds ago. The driver has not received any packets from the server.
INFO|22986/0|10-12-09 15:56:04|MyError:stmt is null in finally close
INFO|22986/0|10-12-09 15:56:04|MyError:dbconn is null in finally close
INFO|22986/0|10-12-09 15:56:04|session closed: try again
INFO|22986/0|10-12-09 15:56:04|Controller State: WAITING_CLOSED -> USER_STOP
INFO|wrapper|10-12-09 15:56:04|stop config name null
INFO|wrapper|10-12-09 15:56:04|externalStop false
INFO|wrapper|10-12-09 15:56:05|exit code linux process 0
INFO|wrapper|10-12-09 15:56:05|killing 22986
INFO|22986/0|10-12-09 15:56:05|Controller State: USER_STOP -> PROCESS_KILLED
INFO|wrapper|10-12-09 15:56:05|process exit code: 0
INFO|22986/0|10-12-09 15:56:05|gobler execption OUTPUT 22986 null
INFO|wrapper|10-12-09 15:56:05|set state RESTART_STOP->RESTART_WAIT
INFO|22986/0|10-12-09 15:56:05|gobler terminated OUTPUT 22986
INFO|wrapper|10-12-09 15:56:10|created pid file /var/run/wrapper.commServer.pid
INFO|wrapper|10-12-09 15:56:10|set state RESTART_WAIT->RESTART_START
INFO|wrapper|10-12-09 15:56:10|starting Process
INFO|22986/0|10-12-09 15:56:10|Controller State: PROCESS_KILLED -> UNKNOWN
INFO|wrapper|10-12-09 15:56:10|Controller State: UNKNOWN -> WAITING
INFO|wrapper|10-12-09 15:56:10|working dir /usr/local
INFO|wrapper|10-12-09 15:56:10|error initializing script
INFO|wrapper|10-12-09 15:56:10|exec:/usr/java/jdk1.6.0_18/bin/java -classpath /usr/local/yajsw-beta-10.2/./wrapperApp.jar:/usr/local -Xrs -Dwrapper.service=true -Dwrapper.console.visible=false -Dwrapper.visible=false -Dwrapper.pidfile=/var/run/wrapper.commServer.pid -Dwrapper.config=/usr/local/yajsw-beta-10.2/conf/wrapper.conf -Dwrapper.port=15005 -Dwrapper.key=2228179562866095885 -Dwrapper.teeName=2228179562866095885$1291881370851 -Dwrapper.tmpPath=/tmp org.rzo.yajsw.app.WrapperJVMMain
INFO|wrapper|10-12-09 15:56:11|started process 23653
INFO|wrapper|10-12-09 15:56:11|started process with pid 23653
INFO|wrapper|10-12-09 15:56:11|set state RESTART_START->RUNNING
FINEST|23653/1|10-12-09 15:56:11| at java.io.FileOutputStream.openAppend(Native Method)
FINEST|23653/1|10-12-09 15:56:11| at java.io.FileOutputStream.<init>(FileOutputStream.java:177)
FINEST|23653/1|10-12-09 15:56:11| at java.io.FileOutputStream.<init>(FileOutputStream.java:102)
FINEST|23653/1|10-12-09 15:56:11| at java.io.FileWriter.<init>(FileWriter.java:61)
FINEST|23653/1|10-12-09 15:56:11| at ConnectionHandler.run(commServer.java:139)
FINEST|23653/1|10-12-09 15:56:11| at java.lang.Thread.run(Thread.java:619)
INFO|23653/1|10-12-09 15:56:11|gobler execption ERROR 22986 Stream closed
INFO|23653/1|10-12-09 15:56:11|gobler terminated ERROR 22986
INFO|wrapper|10-12-09 15:56:25|Controller State: WAITING -> STARTUP_TIMEOUT
INFO|wrapper|10-12-09 15:56:25|restart process due to default exit code rule
INFO|wrapper|10-12-09 15:56:25|set state RUNNING->RESTART
INFO|wrapper|10-12-09 15:56:25|restarting 2 time
INFO|wrapper|10-12-09 15:56:25|set state RESTART->RESTART_STOP
INFO|wrapper|10-12-09 15:56:25|stopping process with pid/timeout 23653 45000
INFO|wrapper|10-12-09 15:56:25|Controller State: STARTUP_TIMEOUT -> USER_STOP
INFO|wrapper|10-12-09 15:56:25|stop config name null
INFO|wrapper|10-12-09 15:56:25|externalStop false
INFO|wrapper|10-12-09 15:57:10|process did not stop after 45000 sec. -> hard kill
INFO|wrapper|10-12-09 15:57:10|killing 23653
INFO|wrapper|10-12-09 15:57:10|send kill sig
INFO|wrapper|10-12-09 15:57:10|exit code linux process 9
INFO|wrapper|10-12-09 15:57:10|Controller State: USER_STOP -> PROCESS_KILLED
INFO|23653/1|10-12-09 15:57:11|gobler execption ERROR 23653 null
INFO|wrapper|10-12-09 15:57:11|process exit code: 999
INFO|23653/1|10-12-09 15:57:11|gobler terminated ERROR 23653
INFO|wrapper|10-12-09 15:57:11|set state RESTART_STOP->RESTART_WAIT
INFO|23653/1|10-12-09 15:57:11|gobler execption OUTPUT 23653 null
INFO|23653/1|10-12-09 15:57:11|gobler terminated OUTPUT 23653
INFO|wrapper|10-12-09 15:57:16|created pid file /var/run/wrapper.commServer.pid
INFO|wrapper|10-12-09 15:57:16|set state RESTART_WAIT->RESTART_START
INFO|wrapper|10-12-09 15:57:16|starting Process
INFO|wrapper|10-12-09 15:57:16|Controller State: PROCESS_KILLED -> UNKNOWN
INFO|wrapper|10-12-09 15:57:16|Controller State: UNKNOWN -> WAITING
INFO|wrapper|10-12-09 15:57:16|working dir /usr/local
INFO|wrapper|10-12-09 15:57:16|error initializing script
INFO|wrapper|10-12-09 15:57:16|exec:/usr/java/jdk1.6.0_18/bin/java -classpath /usr/local/yajsw-beta-10.2/./wrapperApp.jar:/usr/local -Xrs -Dwrapper.service=true -Dwrapper.console.visible=false -Dwrapper.visible=false -Dwrapper.pidfile=/var/run/wrapper.commServer.pid -Dwrapper.config=/usr/local/yajsw-beta-10.2/conf/wrapper.conf -Dwrapper.port=15005 -Dwrapper.key=2228179562866095885 -Dwrapper.teeName=2228179562866095885$1291881436518 -Dwrapper.tmpPath=/tmp org.rzo.yajsw.app.WrapperJVMMain
INFO|wrapper|10-12-09 15:57:17|started process 23655
INFO|wrapper|10-12-09 15:57:17|started process with pid 23655
INFO|wrapper|10-12-09 15:57:17|set state RESTART_START->RUNNING
INFO|wrapper|10-12-09 15:57:31|Controller State: WAITING -> STARTUP_TIMEOUT
INFO|wrapper|10-12-09 15:57:31|restart process due to default exit code rule
INFO|wrapper|10-12-09 15:57:31|set state RUNNING->RESTART
INFO|wrapper|10-12-09 15:57:31|restarting 3 time
INFO|wrapper|10-12-09 15:57:31|set state RESTART->RESTART_STOP
INFO|wrapper|10-12-09 15:57:31|stopping process with pid/timeout 23655 45000
INFO|wrapper|10-12-09 15:57:31|Controller State: STARTUP_TIMEOUT -> USER_STOP
INFO|wrapper|10-12-09 15:57:31|stop config name null
INFO|wrapper|10-12-09 15:57:31|externalStop false
INFO|wrapper|10-12-09 15:58:16|process did not stop after 45000 sec. -> hard kill
INFO|wrapper|10-12-09 15:58:16|killing 23655
INFO|wrapper|10-12-09 15:58:16|send kill sig
INFO|wrapper|10-12-09 15:58:16|exit code linux process 9
INFO|wrapper|10-12-09 15:58:16|Controller State: USER_STOP -> PROCESS_KILLED
INFO|23655/2|10-12-09 15:58:17|gobler execption OUTPUT 23655 null
INFO|wrapper|10-12-09 15:58:17|process exit code: 999
INFO|wrapper|10-12-09 15:58:17|set state RESTART_STOP->RESTART_WAIT
INFO|23655/2|10-12-09 15:58:17|gobler terminated OUTPUT 23655
INFO|23655/2|10-12-09 15:58:17|gobler execption ERROR 23655 null
INFO|23655/2|10-12-09 15:58:17|gobler terminated ERROR 23655
INFO|wrapper|10-12-09 15:58:22|created pid file /var/run/wrapper.commServer.pid
INFO|wrapper|10-12-09 15:58:22|set state RESTART_WAIT->RESTART_START
INFO|wrapper|10-12-09 15:58:22|starting Process
INFO|wrapper|10-12-09 15:58:22|Controller State: PROCESS_KILLED -> UNKNOWN
INFO|wrapper|10-12-09 15:58:22|Controller State: UNKNOWN -> WAITING
INFO|wrapper|10-12-09 15:58:22|working dir /usr/local
INFO|wrapper|10-12-09 15:58:22|error initializing script
INFO|wrapper|10-12-09 15:58:22|exec:/usr/java/jdk1.6.0_18/bin/java -classpath /usr/local/yajsw-beta-10.2/./wrapperApp.jar:/usr/local -Xrs -Dwrapper.service=true -Dwrapper.console.visible=false -Dwrapper.visible=false -Dwrapper.pidfile=/var/run/wrapper.commServer.pid -Dwrapper.config=/usr/local/yajsw-beta-10.2/conf/wrapper.conf -Dwrapper.port=15005 -Dwrapper.key=2228179562866095885 -Dwrapper.teeName=2228179562866095885$1291881502179 -Dwrapper.tmpPath=/tmp org.rzo.yajsw.app.WrapperJVMMain

Actually if you see I have put this ex.printStackTrace(); but maybe is not capture by the wrapper log file. So do you think if I change it to this method will it work

StringWriter sw = new StringWriter();
    PrintWriter pw = new PrintWriter(sw);
    ex.printStackTrace(pw);

Ok I will listen your advice in the last part I will close r then what else must I close. Must I close this receivedSocketConn1?

About the refactoring give me some time to absorb it. I will look through and do it and let you have a look. Thank you.

So do you think if I change it to this method will it work

OK, since it seems only STDOUT is captured by the wrapper code, a better version of your code would be:

e.printStackTrace(System.out)

Ok I will listen your advice in the last part I will close r then what else must I close. Must I close this receivedSocketConn1?

Yes, make it a rule to *always* close all resources you open. It's a good practice anyways, much better than keeping track of what gets closed automatically.

About the refactoring give me some time to absorb it. I will look through and do it and let you have a look

Take your time, one step at a time. First think of splitting all that logic in methods. Good luck.

Thank you

HTH

Edited 5 Years Ago by ~s.o.s~: n/a

OK, I just looked at the source code of the Socket classes and it seems that closing any one of those three (InputStream, OutputStream, Socket) terminates the connection so that really shouldn't be a problem in your case. But the error message you posted really implies that your system has run out of file descriptors to allocate. If you run into this problem again, make sure you get the entire stack trace for it (based on my previous suggestion of using e.printStackTrace(System.out) .

Another thing you should do is find out the current open file descriptor limit on your machine and see the number of file descriptors used when your server process is running. For all we know, it might be a genuine issue of 'low defaults'. I'm no pro-system admin so you might also want to consult your system administrator for the same.

Dear Sos,
Actually in linux there few different file descriptor command one is ulimit-n. So in my case which one is the correct method to use to determine that my program is using. Ok then about the closing I will leave the w.close to take the rest as it is.

Read this and this; but as mentioned in my previous posts, always make sure you close *everything* (good practice, remember?) and consult your sysadmin before making any changes.

Dear Sos,
Thank you for the links. So according to your previous reply you said closing any one anyone wil be sufficientt so I think for now I will leave that and concentrate on the e.printStackTrace(System.out). I have put e.printStackTrace(System.out) into each of my catch statements. Do you it will be printed into the wrapper log file?

Yeah, based on the behaviour of the log file you posted, it seems only STDOUT is captured which means you can't use the normal e.printStackTrace() (since it redirects to STDERR).

Re-throwing the exception and handling them at a single place and using a logging library like log4j would be a good idea, but maybe leave those things when you get time. For the time being, just concentrate on capturing the stack traces and monitoring the file descriptor usage/limit on your system.

Dear Sos,
Thank you for your help so far. I will monitor to get the full printStackTrace then I will get back to you.

Dear Sos,
Ok it happened again below is my latest codes and below is the log file entry which show where there error is happening. What I notice is showing this error at commServer.main(commServer.java:29). So meaning it is referring to this line Socket socketConn1 = serverSocketConn.accept();. So what do you think is the problem?

import java.io.*;
import java.net.*;
import java.util.*;
import java.util.Date;
import java.text.*;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.Statement;
import java.sql.SQLException;
import java.sql.*;



public class commServer {
  
  
   public static void main(String[] args) {

      try {
			    final ServerSocket serverSocketConn = new ServerSocket(9000);
				
				while (true) 
					{
						try 
						{
					            Socket socketConn1 = serverSocketConn.accept();
                                new Thread(new ConnectionHandler(socketConn1)).start();			            
						}
						catch(Exception e)
						{
							System.out.println("MyError:Socket Accepting has been caught in main loop."+e.toString());
						    e.printStackTrace(System.out);
						}
					}
			
      } 
      catch (Exception e) 
      {
         System.out.println("MyError:Socket Conn has been caught in main loop."+e.toString());
         e.printStackTrace(System.out);
         //System.exit(0); 
      }
   }
   
}
  class ConnectionHandler implements Runnable {
    private Socket receivedSocketConn1;
    DateFormat dateFormat = new SimpleDateFormat("dd/MM/yyyy HH:mm:ss");
    DateFormat formatter = new SimpleDateFormat("EEE, dd MMM yyyy"); 
    DateFormat inDf=new SimpleDateFormat("ddMMyyHHmmss");  
    DateFormat outDf=new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); 
  

    ConnectionHandler(Socket receivedSocketConn1) {
      this.receivedSocketConn1=receivedSocketConn1;
    }

 
   //@Override
   public void run() { 
      
      Connection dbconn = null;
      BufferedWriter w = null;
      BufferedReader r = null;
      

      try {
      
         PrintStream out = System.out; 
      	 BufferedWriter fout = null;
         w =  new BufferedWriter(new OutputStreamWriter(receivedSocketConn1.getOutputStream()));
         r = new BufferedReader(new InputStreamReader(receivedSocketConn1.getInputStream()));
         
        
         
         
         int m = 0, count=0;
         String line="";
         String n="";
         w.write("$PA\n");
         w.flush();
         while ((m=r.read()) != -1) 
         {
              Date dateIn = new Date();
           	  n = n + (char) m;
           	 
         	  int i = n.indexOf("GET");
						if(i != -1) { 
							break;
						}
        	  
         	  if (m==35)
         	  {
	         	  
	             String ori = n;
	             String noCheckSum =  n.substring(0,(n.length()-4));
     			 int addExist = n.indexOf("@");
     			 
                 String[] slave = null;
                 if(addExist!=-1)
                 {
     	            slave = noCheckSum.split("@");
     	          	n = slave[0];
     	      	    //System.out.println(" slave : "+slave.length);
     	         }
			     else
			     {
			     	n = noCheckSum;
			     }
	                     
	             String[] result = n.split(",");
	                            
	             Statement stmt = null;
	             
	             
	             w.write("$PA\n");
                 w.flush();
	             
      
			      int count1 = 0;
			      
			      
	              	    
                      Date date = Calendar.getInstance().getTime(); 
                      String today = formatter.format(date); 
				      String filename= "MyDataFile"+today+".txt"; 
                      boolean append = true; 
                      
                      FileWriter fw = null;
                      try
                      {
                      
                      fw = new FileWriter(filename,append); 
	                  fw.write(ori+"  "+dateFormat.format(dateIn)+"\n");//appends the string to the file 
	                  Date dateOut = new Date();
	                  fw.write("$PA"+"  "+dateFormat.format(dateOut)+"\n");//appends the string to the file 
                      }
                      catch (IOException ex)  
				      { 
				          //ex.printStackTrace(new PrintWriter(sWriter));
				          System.out.println("MyError:IOException has been caught in in the file operation");
				          ex.printStackTrace(System.out);
				      }  
      				  finally
      				  {
        
        			   try 
       				   {
        
	                   if ( fw != null ) 
	                   {
	          	       fw.close();
	                   }
	                   else 
	                   {
	        	       System.out.println("MyError:fw is null in finally close");
	                   }
       
                       }
				        catch(IOException ex){
				          System.out.println("MyError:IOException has been caught in fw is null in finally close");
				          ex.printStackTrace(System.out);
				
				        }
      				  }
	                  try
			          {
	     	                 	                    				
     					String deviceID=result[3].trim();     	
     					String dateTime=result[4].trim();     					
				     	String[] result2 = result[10].split("'");   				     	
				     	String gpsDate = result2[1].trim().substring(0,6);			     	
				     	String gpsTime = result2[1].trim().substring(6,12);				     	
				     	String gpsLat = result2[1].trim().substring(13,20);
				     	String latitude = result2[1].trim().substring(13,20).substring(0,2)+"."+result2[1].trim().substring(13,20).substring(2,7);
				     	String gpsLong = result2[1].trim().substring(21,29);
				     	String longitude = result2[1].trim().substring(21,29).substring(0,3)+"."+result2[1].trim().substring(21,29).substring(3,8);
				     	String speed = result2[1].trim().substring(30,33);				     	
				     	String course = result2[1].trim().substring(33,36);
			            String dateTimer = null;
					        try
					        { 
					        Date inDate=null;  
					        inDf.setTimeZone(TimeZone.getTimeZone("UTC")); 
					        inDate=inDf.parse(dateTime); 
					        outDf.setTimeZone(TimeZone.getTimeZone("Asia/Kuala_Lumpur")); 
							dateTimer=outDf.format(inDate); 
					        }
					        catch(ParseException ex)
					        { 
					          System.out.println("MyError:Parse Error has been caught for date parse close");
		                      ex.printStackTrace(System.out);
					        } 				 				
				      dbconn = DriverManager.getConnection("jdbc:mysql://192.168.1.155:3306/db1?"+"user=db1&password=test1");
			          stmt = dbconn.createStatement();
			          
			          
			          String selectQuery2 = "Select * from tripData Where deviceID='"+ deviceID +"' and dateTimer>'"+dateTimer+"' Order By dateTimer Desc Limit 1"; 
			          ResultSet rs2 = stmt.executeQuery(selectQuery2);
		
			          String updateQuery = "";
			          if(rs2.next())
			          {
			          	String previousLatitude="";
			          	String previousLongitude="";
			          	String previousSpeed="";
				       
				        previousLatitude = rs2.getString("latitude");
				        previousLongitude = rs2.getString("longitude");
				        previousSpeed = rs2.getString("speed");
				        
				        updateQuery = "UPDATE device SET " + 
							"latitude='" + previousLatitude + 
							"',longitude='" + previousLongitude + 
							"',speed='" + previousSpeed + 
							"' WHERE serialNumber='" + deviceID + "'";
			          }
			          else
			          {
			          
			          updateQuery = "UPDATE device SET " + 
							"latitude='" + latitude + 
							"',longitude='" + longitude + 
							"',speed='" + speed + 
							"',course='" + course +
							"',dateTimer='" + dateTimer +
							"' WHERE serialNumber='" + deviceID + "'";
			          }	
				      count = stmt.executeUpdate(updateQuery);				      
				       
			          String insertQuery = "INSERT INTO tripData" + 
							"(latitude,longitude,speed,course,dateTimer,deviceID)" + " VALUES ('" + 
							latitude + "','" + longitude + "','" + speed + "','" + course + "','" + dateTimer + "','"  + deviceID + "' )";
	                  count = stmt.executeUpdate(insertQuery);
	                  
		                  if(addExist!=-1)
	                      {
		                  for(int iSlave=1; iSlave<slave.length ; iSlave++)
							{
								String[] slaveDetails = slave[iSlave].split(",",-1);
								String slaveEventType = slaveDetails[0];
								String slaveGroup = slaveDetails[1];
								String slaveUnitID = slaveDetails[2];
								String slaveBattLevel = slaveDetails[3];
								String slaveSwitchStat = slaveDetails[4];
								String slaveTempHumid = slaveDetails[5];
							    String slaveTemp="",slaveHumid="";							    
							    if(slaveTempHumid.length()>0)
							    {
							        slaveHumid = slaveTempHumid.substring(5,7);
									if(slaveTempHumid.charAt(0)=='P')
									{
  									slaveTemp = "+"+slaveTempHumid.substring(1,3)+"."+slaveTempHumid.substring(3,5);										
									}
									else if(slaveTempHumid.charAt(0)=='M')
									{
									    slaveTemp = "-"+slaveTempHumid.substring(1,3)+"."+slaveTempHumid.substring(3,5);
									}
							    }
							    slaveBattLevel = slaveBattLevel.trim().substring(0,2)+"."+slaveBattLevel.trim().substring(2,3);
								String insertQuery2 = "INSERT INTO tripDataSlave" + 
								"(dateTimer,deviceID,slaveUnitID,slaveEventType,slaveGroup,slaveBattLevel,slaveSwitchStat,slaveTemp,slaveHumidity)" + " VALUES ('" + 
								dateTimer + "','" + deviceID + "','" + slaveUnitID + "','" + slaveEventType + "','" + slaveGroup + "','" + slaveBattLevel + "','" + slaveSwitchStat + "','" + slaveTemp + "','" + slaveHumid + "')";
		                        count = stmt.executeUpdate(insertQuery2);
							}
	                      }
			          }
			          catch (SQLException ex)  
			          { 
                         System.out.println("MyError:Error : "+ex);
                         ex.printStackTrace(System.out);

			          }      
				      finally
				      {
				        try 
				       	{
					        if ( stmt != null ) 
					        {
					          stmt.close();
					        }
					        else 
					        {
					        	System.out.println("MyError:stmt is null in finally close");
					        }
				        }
				        catch(SQLException ex){
				            System.out.println("MyError:SQLException has been caught for stmt close");
                            ex.printStackTrace(System.out);
				        }
				        try 
				       	{
					        if ( dbconn != null ) 
					        {
					          dbconn.close();
					        }
					        else 
					        {
					          System.out.println("MyError:dbconn is null in finally close");
					        }
				        }
				        catch(SQLException ex){
				             System.out.println("MyError:SQLException has been caught for dbconn close");
                             ex.printStackTrace(System.out);
				        }
								        
				      }
	             n="";
         	  }
         }
      } 
      catch (IOException ex)  
      { 
           System.out.println("MyError:IOException has been caught in in the main first try");
           ex.printStackTrace(System.out);
      }      
      finally
      {
        try 
       	{
	        if ( w != null ) 
	        {
	          	w.close();
	        }
	        else 
	        {
	        	System.out.println("MyError:w is null in finally close");
	        }
        }
        catch(IOException ex){
           System.out.println("MyError:IOException has been caught in w in finally close");
           ex.printStackTrace(System.out);
        }
        
      }
   }
}

Log file entry

FINEST|7155/0|10-12-14 14:51:19|MyError:Socket Accepting has been caught in main loop.java.net.SocketException: Too many open files
INFO|wrapper|10-12-14 14:51:19|Trigger found: Too many open files
INFO|wrapper|10-12-14 14:51:19|set state RUNNING->RESTART
INFO|wrapper|10-12-14 14:51:19|restarting 1 time
INFO|wrapper|10-12-14 14:51:19|set state RESTART->RESTART_STOP
INFO|wrapper|10-12-14 14:51:19|stopping process with pid/timeout 7155 45000
FINEST|7155/0|10-12-14 14:51:19|Dec 14, 2010 2:51:19 PM org.rzo.yajsw.nettyutils.LoggingFilter log
INFO|7155/0|10-12-14 14:51:19|Controller State: LOGGED_ON -> WAITING_CLOSED
FINEST|7155/0|10-12-14 14:51:19|INFO: [app][id: 0x72e5355f, /127.0.0.1:45789 => /127.0.0.1:15003] RECEIVED: BigEndianHeapChannelBuffer(ridx=0, widx=2, cap=2)
FINEST|7155/0|10-12-14 14:51:19|Dec 14, 2010 2:51:19 PM org.rzo.yajsw.nettyutils.LoggingFilter log
FINEST|7155/0|10-12-14 14:51:19|INFO: [app][id: 0x72e5355f, /127.0.0.1:45789 => /127.0.0.1:15003] RECEIVED: BigEndianHeapChannelBuffer(ridx=0, widx=2, cap=2)
FINEST|7155/0|10-12-14 14:51:19|Dec 14, 2010 2:51:19 PM org.rzo.yajsw.nettyutils.LoggingFilter log
FINEST|7155/0|10-12-14 14:51:19|INFO: [app][id: 0x72e5355f, /127.0.0.1:45789 => /127.0.0.1:15003] CLOSE
FINEST|7155/0|10-12-14 14:51:19|Dec 14, 2010 2:51:19 PM org.rzo.yajsw.nettyutils.LoggingFilter log
FINEST|7155/0|10-12-14 14:51:19|INFO: [app][id: 0x72e5355f, /127.0.0.1:45789 => /127.0.0.1:15003] DISCONNECTED
FINEST|7155/0|10-12-14 14:51:19|Dec 14, 2010 2:51:19 PM org.rzo.yajsw.nettyutils.LoggingFilter log
FINEST|7155/0|10-12-14 14:51:19|INFO: [app][id: 0x72e5355f, /127.0.0.1:45789 => /127.0.0.1:15003] UNBOUND
FINEST|7155/0|10-12-14 14:51:19|Dec 14, 2010 2:51:19 PM org.rzo.yajsw.nettyutils.LoggingFilter log
FINEST|7155/0|10-12-14 14:51:19|INFO: [app][id: 0x72e5355f, /127.0.0.1:45789 => /127.0.0.1:15003] CLOSED
INFO|7155/0|10-12-14 14:51:19|Controller State: WAITING_CLOSED -> USER_STOP
INFO|wrapper|10-12-14 14:51:19|stop config name null
INFO|wrapper|10-12-14 14:51:19|externalStop false
INFO|wrapper|10-12-14 14:51:20|exit code linux process 0
INFO|7155/0|10-12-14 14:51:20|Controller State: USER_STOP -> PROCESS_KILLED
INFO|wrapper|10-12-14 14:51:20|killing 7155
INFO|7155/0|10-12-14 14:51:20|gobler execption ERROR 7155 null
INFO|7155/0|10-12-14 14:51:20|gobler terminated ERROR 7155
INFO|wrapper|10-12-14 14:51:20|process exit code: 0
INFO|wrapper|10-12-14 14:51:20|set state RESTART_STOP->RESTART_WAIT
INFO|wrapper|10-12-14 14:51:25|created pid file /var/run/wrapper.commServer.pid
INFO|wrapper|10-12-14 14:51:25|set state RESTART_WAIT->RESTART_START
INFO|wrapper|10-12-14 14:51:25|starting Process
INFO|7155/0|10-12-14 14:51:25|Controller State: PROCESS_KILLED -> UNKNOWN
INFO|wrapper|10-12-14 14:51:25|Controller State: UNKNOWN -> WAITING
INFO|wrapper|10-12-14 14:51:25|working dir /usr/local
INFO|wrapper|10-12-14 14:51:25|error initializing script
INFO|wrapper|10-12-14 14:51:25|exec:/usr/java/jdk1.6.0_18/bin/java -classpath /usr/local/yajsw-beta-10.2/./wrapperApp.jar:/usr/local -Xrs -Dwrapper.service=true -Dwrapper.console.visible=false -Dwrapper.visible=false -Dwrapper.pidfile=/var/run/wrapper.commServer.pid -Dwrapper.config=/usr/local/yajsw-beta-10.2/conf/wrapper.conf -Dwrapper.port=15003 -Dwrapper.key=-4953607583810484427 -Dwrapper.teeName=-4953607583810484427$1292309485955 -Dwrapper.tmpPath=/tmp org.rzo.yajsw.app.WrapperJVMMain
INFO|wrapper|10-12-14 14:51:26|started process 14438
INFO|wrapper|10-12-14 14:51:26|started process with pid 14438
INFO|wrapper|10-12-14 14:51:26|set state RESTART_START->RUNNING
FINEST|14438/1|10-12-14 14:51:26|java.net.SocketException: Too many open files
INFO|wrapper|10-12-14 14:51:26|Trigger found: Too many open files
INFO|wrapper|10-12-14 14:51:26|set state RUNNING->RESTART
INFO|wrapper|10-12-14 14:51:26|restarting 2 time
INFO|wrapper|10-12-14 14:51:26|set state RESTART->RESTART_STOP
INFO|wrapper|10-12-14 14:51:26|stopping process with pid/timeout 14438 45000
INFO|wrapper|10-12-14 14:51:26|Controller State: WAITING -> USER_STOP
INFO|wrapper|10-12-14 14:51:26|stop config name null
INFO|wrapper|10-12-14 14:51:26|externalStop false
INFO|wrapper|10-12-14 14:52:11|process did not stop after 45000 sec. -> hard kill
INFO|wrapper|10-12-14 14:52:11|killing 14438
INFO|wrapper|10-12-14 14:52:11|send kill sig
INFO|wrapper|10-12-14 14:52:11|exit code linux process 9
INFO|wrapper|10-12-14 14:52:11|Controller State: USER_STOP -> PROCESS_KILLED
INFO|14438/1|10-12-14 14:52:12|gobler execption OUTPUT 14438 null
INFO|wrapper|10-12-14 14:52:12|process exit code: 999
INFO|14438/1|10-12-14 14:52:12|gobler terminated OUTPUT 14438
INFO|14438/1|10-12-14 14:52:12|gobler execption ERROR 14438 null
INFO|wrapper|10-12-14 14:52:12|set state RESTART_STOP->RESTART_WAIT
INFO|14438/1|10-12-14 14:52:12|gobler terminated ERROR 14438
INFO|wrapper|10-12-14 14:52:17|created pid file /var/run/wrapper.commServer.pid
INFO|wrapper|10-12-14 14:52:17|set state RESTART_WAIT->RESTART_START
INFO|wrapper|10-12-14 14:52:17|starting Process
INFO|wrapper|10-12-14 14:52:17|Controller State: PROCESS_KILLED -> UNKNOWN
INFO|wrapper|10-12-14 14:52:17|Controller State: UNKNOWN -> WAITING
INFO|wrapper|10-12-14 14:52:17|working dir /usr/local
INFO|wrapper|10-12-14 14:52:17|error initializing script
INFO|wrapper|10-12-14 14:52:17|exec:/usr/java/jdk1.6.0_18/bin/java -classpath /usr/local/yajsw-beta-10.2/./wrapperApp.jar:/usr/local -Xrs -Dwrapper.service=true -Dwrapper.console.visible=false -Dwrapper.visible=false -Dwrapper.pidfile=/var/run/wrapper.commServer.pid -Dwrapper.config=/usr/local/yajsw-beta-10.2/conf/wrapper.conf -Dwrapper.port=15003 -Dwrapper.key=-4953607583810484427 -Dwrapper.teeName=-4953607583810484427$1292309537216 -Dwrapper.tmpPath=/tmp org.rzo.yajsw.app.WrapperJVMMain
INFO|wrapper|10-12-14 14:52:17|started process 14440
INFO|wrapper|10-12-14 14:52:17|started process with pid 14440
INFO|wrapper|10-12-14 14:52:17|set state RESTART_START->RUNNING
FINEST|14440/2|10-12-14 14:52:17| at java.net.PlainSocketImpl.socketAccept(Native Method)
FINEST|14440/2|10-12-14 14:52:17| at java.net.PlainSocketImpl.accept(PlainSocketImpl.java:390)
FINEST|14440/2|10-12-14 14:52:17| at java.net.ServerSocket.implAccept(ServerSocket.java:453)
FINEST|14440/2|10-12-14 14:52:17| at java.net.ServerSocket.accept(ServerSocket.java:421)
FINEST|14440/2|10-12-14 14:52:17| at commServer.main(commServer.java:29)
FINEST|14440/2|10-12-14 14:52:17| at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
FINEST|14440/2|10-12-14 14:52:17| at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
FINEST|14440/2|10-12-14 14:52:17| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
FINEST|14440/2|10-12-14 14:52:17| at java.lang.reflect.Method.invoke(Method.java:597)
FINEST|14440/2|10-12-14 14:52:17| at org.rzo.yajsw.app.WrapperJVMMain.executeMain(WrapperJVMMain.java:53)
FINEST|14440/2|10-12-14 14:52:17| at org.rzo.yajsw.app.WrapperJVMMain.main(WrapperJVMMain.java:36)
FINEST|14440/2|10-12-14 14:52:17|MyError:Socket Accepting has been caught in main loop.java.net.SocketException: Too many open files
INFO|wrapper|10-12-14 14:52:17|Trigger found: Too many open files
INFO|wrapper|10-12-14 14:52:17|set state RUNNING->RESTART
INFO|wrapper|10-12-14 14:52:17|restarting 3 time
INFO|wrapper|10-12-14 14:52:17|set state RESTART->RESTART_STOP
INFO|wrapper|10-12-14 14:52:17|stopping process with pid/timeout 14440 45000
INFO|wrapper|10-12-14 14:52:17|Controller State: WAITING -> USER_STOP
INFO|wrapper|10-12-14 14:52:17|stop config name null
INFO|wrapper|10-12-14 14:52:17|externalStop false
INFO|wrapper|10-12-14 14:53:02|process did not stop after 45000 sec. -> hard kill
INFO|wrapper|10-12-14 14:53:02|killing 14440
INFO|wrapper|10-12-14 14:53:02|send kill sig
INFO|wrapper|10-12-14 14:53:02|exit code linux process 9
INFO|wrapper|10-12-14 14:53:02|Controller State: USER_STOP -> PROCESS_KILLED
INFO|14440/2|10-12-14 14:53:03|gobler execption OUTPUT 14440 null
INFO|14440/2|10-12-14 14:53:03|gobler execption ERROR 14440 null
INFO|wrapper|10-12-14 14:53:03|process exit code: 999
INFO|14440/2|10-12-14 14:53:03|gobler terminated ERROR 14440
INFO|14440/2|10-12-14 14:53:03|gobler terminated OUTPUT 14440
INFO|wrapper|10-12-14 14:53:03|set state RESTART_STOP->RESTART_WAIT
INFO|wrapper|10-12-14 14:53:08|created pid file /var/run/wrapper.commServer.pid
INFO|wrapper|10-12-14 14:53:08|set state RESTART_WAIT->RESTART_START
INFO|wrapper|10-12-14 14:53:08|starting Process
INFO|wrapper|10-12-14 14:53:08|Controller State: PROCESS_KILLED -> UNKNOWN
INFO|wrapper|10-12-14 14:53:08|Controller State: UNKNOWN -> WAITING
INFO|wrapper|10-12-14 14:53:08|working dir /usr/local
INFO|wrapper|10-12-14 14:53:08|error initializing script
INFO|wrapper|10-12-14 14:53:08|exec:/usr/java/jdk1.6.0_18/bin/java -classpath /usr/local/yajsw-beta-10.2/./wrapperApp.jar:/usr/local -Xrs -Dwrapper.service=true -Dwrapper.console.visible=false -Dwrapper.visible=false -Dwrapper.pidfile=/var/run/wrapper.commServer.pid -Dwrapper.config=/usr/local/yajsw-beta-10.2/conf/wrapper.conf -Dwrapper.port=15003 -Dwrapper.key=-4953607583810484427 -Dwrapper.teeName=-4953607583810484427$1292309588467 -Dwrapper.tmpPath=/tmp org.rzo.yajsw.app.WrapperJVMMain
INFO|wrapper|10-12-14 14:53:08|started process 14441
INFO|wrapper|10-12-14 14:53:08|started process with pid 14441
INFO|wrapper|10-12-14 14:53:08|set state RESTART_START->RUNNING
FINEST|14441/3|10-12-14 14:53:08|java.net.SocketException: Too many open files
INFO|wrapper|10-12-14 14:53:08|Trigger found: Too many open files
INFO|wrapper|10-12-14 14:53:08|set state RUNNING->RESTART
INFO|wrapper|10-12-14 14:53:08|restarting 4 time
INFO|wrapper|10-12-14 14:53:08|set state RESTART->RESTART_STOP
INFO|wrapper|10-12-14 14:53:08|stopping process with pid/timeout 14441 45000
INFO|wrapper|10-12-14 14:53:08|Controller State: WAITING -> USER_STOP
INFO|wrapper|10-12-14 14:53:08|stop config name null
INFO|wrapper|10-12-14 14:53:08|externalStop false
INFO|wrapper|10-12-14 14:53:53|process did not stop after 45000 sec. -> hard kill
INFO|wrapper|10-12-14 14:53:53|killing 14441
INFO|wrapper|10-12-14 14:53:53|send kill sig
INFO|wrapper|10-12-14 14:53:53|exit code linux process 9
INFO|wrapper|10-12-14 14:53:54|Controller State: USER_STOP -> PROCESS_KILLED
INFO|14441/3|10-12-14 14:53:54|gobler execption ERROR 14441 null
INFO|wrapper|10-12-14 14:53:54|process exit code: 999
INFO|14441/3|10-12-14 14:53:54|gobler execption OUTPUT 14441 null
INFO|14441/3|10-12-14 14:53:54|gobler terminated ERROR 14441
INFO|wrapper|10-12-14 14:53:54|set state RESTART_STOP->RESTART_WAIT
INFO|14441/3|10-12-14 14:53:54|gobler terminated OUTPUT 14441
INFO|wrapper|10-12-14 14:53:59|created pid file /var/run/wrapper.commServer.pid
INFO|wrapper|10-12-14 14:53:59|set state RESTART_WAIT->RESTART_START
INFO|wrapper|10-12-14 14:53:59|starting Process
INFO|wrapper|10-12-14 14:53:59|Controller State: PROCESS_KILLED -> UNKNOWN
INFO|wrapper|10-12-14 14:53:59|Controller State: UNKNOWN -> WAITING
INFO|wrapper|10-12-14 14:53:59|working dir /usr/local
INFO|wrapper|10-12-14 14:53:59|error initializing script
INFO|wrapper|10-12-14 14:53:59|exec:/usr/java/jdk1.6.0_18/bin/java -classpath /usr/local/yajsw-beta-10.2/./wrapperApp.jar:/usr/local -Xrs -Dwrapper.service=true -Dwrapper.console.visible=false -Dwrapper.visible=false -Dwrapper.pidfile=/var/run/wrapper.commServer.pid -Dwrapper.config=/usr/local/yajsw-beta-10.2/conf/wrapper.conf -Dwrapper.port=15003 -Dwrapper.key=-4953607583810484427 -Dwrapper.teeName=-4953607583810484427$1292309639718 -Dwrapper.tmpPath=/tmp org.rzo.yajsw.app.WrapperJVMMain
INFO|wrapper|10-12-14 14:54:00|started process 14442
INFO|wrapper|10-12-14 14:54:00|started process with pid 14442
INFO|wrapper|10-12-14 14:54:00|set state RESTART_START->RUNNING
FINEST|14442/4|10-12-14 14:54:00| at java.net.PlainSocketImpl.socketAccept(Native Method)
FINEST|14442/4|10-12-14 14:54:00| at java.net.PlainSocketImpl.accept(PlainSocketImpl.java:390)
FINEST|14442/4|10-12-14 14:54:00| at java.net.ServerSocket.implAccept(ServerSocket.java:453)
FINEST|14442/4|10-12-14 14:54:00| at java.net.ServerSocket.accept(ServerSocket.java:421)
FINEST|14442/4|10-12-14 14:54:00| at commServer.main(commServer.java:29)
FINEST|14442/4|10-12-14 14:54:00| at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
FINEST|14442/4|10-12-14 14:54:00| at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
FINEST|14442/4|10-12-14 14:54:00| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
FINEST|14442/4|10-12-14 14:54:00| at java.lang.reflect.Method.invoke(Method.java:597)
FINEST|14442/4|10-12-14 14:54:00| at org.rzo.yajsw.app.WrapperJVMMain.executeMain(WrapperJVMMain.java:53)
FINEST|14442/4|10-12-14 14:54:00| at org.rzo.yajsw.app.WrapperJVMMain.main(WrapperJVMMain.java:36)
FINEST|14442/4|10-12-14 14:54:00|MyError:Socket Accepting has been caught in main loop.java.net.SocketException: Too many open files
INFO|wrapper|10-12-14 14:54:00|Trigger found: Too many open files
INFO|wrapper|10-12-14 14:54:00|set state RUNNING->RESTART
INFO|wrapper|10-12-14 14:54:00|restarting 5 time
INFO|wrapper|10-12-14 14:54:00|set state RESTART->RESTART_STOP
INFO|wrapper|10-12-14 14:54:00|stopping process with pid/timeout 14442 45000
INFO|wrapper|10-12-14 14:54:00|Controller State: WAITING -> USER_STOP
INFO|wrapper|10-12-14 14:54:00|stop config name null
INFO|wrapper|10-12-14 14:54:00|externalStop false
INFO|wrapper|10-12-14 14:54:45|process did not stop after 45000 sec. -> hard kill
INFO|wrapper|10-12-14 14:54:45|killing 14442
INFO|wrapper|10-12-14 14:54:45|send kill sig
INFO|wrapper|10-12-14 14:54:45|exit code linux process 9
INFO|wrapper|10-12-14 14:54:45|Controller State: USER_STOP -> PROCESS_KILLED
INFO|14442/4|10-12-14 14:54:45|gobler execption ERROR 14442 null
INFO|14442/4|10-12-14 14:54:45|gobler execption OUTPUT 14442 null
INFO|wrapper|10-12-14 14:54:45|process exit code: 999
INFO|14442/4|10-12-14 14:54:45|gobler terminated OUTPUT 14442
INFO|14442/4|10-12-14 14:54:45|gobler terminated ERROR 14442
INFO|wrapper|10-12-14 14:54:45|set state RESTART_STOP->RESTART_WAIT
INFO|wrapper|10-12-14 14:54:50|created pid file /var/run/wrapper.commServer.pid
INFO|wrapper|10-12-14 14:54:50|set state RESTART_WAIT->RESTART_START
INFO|wrapper|10-12-14 14:54:50|starting Process
INFO|wrapper|10-12-14 14:54:50|Controller State: PROCESS_KILLED -> UNKNOWN
INFO|wrapper|10-12-14 14:54:50|Controller State: UNKNOWN -> WAITING
INFO|wrapper|10-12-14 14:54:50|working dir /usr/local
INFO|wrapper|10-12-14 14:54:50|error initializing script
INFO|wrapper|10-12-14 14:54:50|exec:/usr/java/jdk1.6.0_18/bin/java -classpath /usr/local/yajsw-beta-10.2/./wrapperApp.jar:/usr/local -Xrs -Dwrapper.service=true -Dwrapper.console.visible=false -Dwrapper.visible=false -Dwrapper.pidfile=/var/run/wrapper.commServer.pid -Dwrapper.config=/usr/local/yajsw-beta-10.2/conf/wrapper.conf -Dwrapper.port=15003 -Dwrapper.key=-4953607583810484427 -Dwrapper.teeName=-4953607583810484427$1292309690969 -Dwrapper.tmpPath=/tmp org.rzo.yajsw.app.WrapperJVMMain
INFO|wrapper|10-12-14 14:54:51|started process 14443
INFO|wrapper|10-12-14 14:54:51|started process with pid 14443
INFO|wrapper|10-12-14 14:54:51|set state RESTART_START->RUNNING
FINEST|14443/5|10-12-14 14:54:51|java.net.SocketException: Too many open files
INFO|wrapper|10-12-14 14:54:51|Trigger found: Too many open files
INFO|wrapper|10-12-14 14:54:51|too many restarts
INFO|wrapper|10-12-14 14:54:51|set state RUNNING->STATE_USER_STOP
INFO|wrapper|10-12-14 14:54:51|stopping process with pid/timeout 14443 45000
INFO|wrapper|10-12-14 14:54:51|Controller State: WAITING -> USER_STOP
INFO|wrapper|10-12-14 14:54:51|stop config name null
INFO|wrapper|10-12-14 14:54:51|externalStop false
INFO|wrapper|10-12-14 14:55:36|process did not stop after 45000 sec. -> hard kill
INFO|wrapper|10-12-14 14:55:36|killing 14443
INFO|wrapper|10-12-14 14:55:36|send kill sig
INFO|wrapper|10-12-14 14:55:36|exit code linux process 9
INFO|wrapper|10-12-14 14:55:36|Controller State: USER_STOP -> PROCESS_KILLED
INFO|wrapper|10-12-14 14:55:36|giving up after 5 retries
INFO|14443/5|10-12-14 14:55:37|gobler execption OUTPUT 14443 null
INFO|wrapper|10-12-14 14:55:37|process exit code: 999
INFO|14443/5|10-12-14 14:55:37|gobler execption ERROR 14443 null
INFO|wrapper|10-12-14 14:55:37|set state STATE_USER_STOP->IDLE
INFO|14443/5|10-12-14 14:55:37|gobler terminated OUTPUT 14443
INFO|14443/5|10-12-14 14:55:37|gobler terminated ERROR 14443
FINEST|14443/5|10-12-14 14:55:37| at java.net.PlainSocketImpl.socketAccept(Native Method)
FINEST|14443/5|10-12-14 14:55:37| at java.net.PlainSocketImpl.accept(PlainSocketImpl.java:390)
FINEST|14443/5|10-12-14 14:55:37| at java.net.ServerSocket.implAccept(ServerSocket.java:453)
FINEST|14443/5|10-12-14 14:55:37| at java.net.ServerSocket.accept(ServerSocket.java:421)
FINEST|14443/5|10-12-14 14:55:37| at commServer.main(commServer.java:29)
FINEST|14443/5|10-12-14 14:55:37| at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
FINEST|14443/5|10-12-14 14:55:37| at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
FINEST|14443/5|10-12-14 14:55:37| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
FINEST|14443/5|10-12-14 14:55:37| at java.lang.reflect.Method.invoke(Method.java:597)
FINEST|14443/5|10-12-14 14:55:37| at org.rzo.yajsw.app.WrapperJVMMain.executeMain(WrapperJVMMain.java:53)
FINEST|14443/5|10-12-14 14:55:37| at org.rzo.yajsw.app.WrapperJVMMain.main(WrapperJVMMain.java:36)
FINEST|14443/5|10-12-14 14:55:37|MyError:Socket Accepting has been caught in main loop.java.net.SocketException: Too many open files
INFO|wrapper|10-12-14 14:55:37|Trigger found: Too many open files
INFO|wrapper|10-12-14 14:55:37|too many restarts
FINEST|14443/5|10-12-14 14:55:37|java.net.SocketException: Too many open files
INFO|wrapper|10-12-14 14:55:37|Trigger found: Too many open files
INFO|wrapper|10-12-14 14:55:37|too many restarts
FINEST|14443/5|10-12-14 14:55:37| at java.net.PlainSocketImpl.socketAccept(Native Method)
FINEST|14443/5|10-12-14 14:55:37| at java.net.PlainSocketImpl.accept(PlainSocketImpl.java:390)
FINEST|14443/5|10-12-14 14:55:37| at java.net.ServerSocket.implAccept(ServerSocket.java:453)
FINEST|14443/5|10-12-14 14:55:37| at java.net.ServerSocket.accept(ServerSocket.java:421)
FINEST|14443/5|10-12-14 14:55:37| at commServer.main(commServer.java:29)
FINEST|14443/5|10-12-14 14:55:37| at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
FINEST|14443/5|10-12-14 14:55:37| at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
FINEST|14443/5|10-12-14 14:55:37| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
FINEST|14443/5|10-12-14 14:55:37| at java.lang.reflect.Method.invoke(Method.java:597)
FINEST|14443/5|10-12-14 14:55:37| at org.rzo.yajsw.app.WrapperJVMMain.executeMain(WrapperJVMMain.java:53)
FINEST|14443/5|10-12-14 14:55:37| at org.rzo.yajsw.app.WrapperJVMMain.main(WrapperJVMMain.java:36)
FINEST|14443/5|10-12-14 14:55:37|MyError:Socket Accepting has been caught in main loop.java.net.SocketException: Too many open files
INFO|wrapper|10-12-14 14:55:37|Trigger found: Too many open files
INFO|wrapper|10-12-14 14:55:37|too many restarts
FINEST|14443/5|10-12-14 14:55:37|java.net.SocketException: Too many open files
INFO|wrapper|10-12-14 14:55:37|Trigger found: Too many open files
INFO|wrapper|10-12-14 14:55:37|too many restarts
FINEST|14443/5|10-12-14 14:55:37| at java.net.PlainSocketImpl.socketAccept(Native Method)
FINEST|14443/5|10-12-14 14:55:37| at java.net.PlainSocketImpl.accept(PlainSocketImpl.java:390)
FINEST|14443/5|10-12-14 14:55:37| at java.net.ServerSocket.implAccept(ServerSocket.java:453)
FINEST|14443/5|10-12-14 14:55:37| at java.net.ServerSocket.accept(ServerSocket.java:421)
FINEST|14443/5|10-12-14 14:55:37| at commServer.main(commServer.java:29)
FINEST|14443/5|10-12-14 14:55:37| at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
FINEST|14443/5|10-12-14 14:55:37| at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
FINEST|14443/5|10-12-14 14:55:37| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
FINEST|14443/5|10-12-14 14:55:37| at java.lang.reflect.Method.invoke(Method.java:597)
FINEST|14443/5|10-12-14 14:55:37| at org.rzo.yajsw.app.WrapperJVMMain.executeMain(WrapperJVMMain.java:53)
FINEST|14443/5|10-12-14 14:55:37| at org.rzo.yajsw.app.WrapperJVMMain.main(WrapperJVMMain.java:36)
FINEST|14443/5|10-12-14 14:55:37|MyError:Socket Accepting has been caught in main loop.java.net.SocketException: Too many open files
INFO|wrapper|10-12-14 14:55:37|Trigger found: Too many open files
INFO|wrapper|10-12-14 14:55:37|too many restarts
FINEST|14443/5|10-12-14 14:55:37|java.net.SocketException: Too many open files
INFO|wrapper|10-12-14 14:55:37|Trigger found: Too many open files
INFO|wrapper|10-12-14 14:55:37|too many restarts
FINEST|14443/5|10-12-14 14:55:37| at java.net.PlainSocketImpl.socketAccept(Native Method)
FINEST|14443/5|10-12-14 14:55:37| at java.net.PlainSocketImpl.accept(PlainSocketImpl.java:390)
FINEST|14443/5|10-12-14 14:55:37| at java.net.ServerSocket.implAccept(ServerSocket.java:453)
FINEST|14443/5|10-12-14 14:55:37| at java.net.ServerSocket.accept(ServerSocket.java:421)
FINEST|14443/5|10-12-14 14:55:37| at commServer.main(commServer.java:29)
FINEST|14443/5|10-12-14 14:55:37| at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
FINEST|14443/5|10-12-14 14:55:37| at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
FINEST|14443/5|10-12-14 14:55:37| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
FINEST|14443/5|10-12-14 14:55:37| at java.lang.reflect.Method.invoke(Method.java:597)
FINEST|14443/5|10-12-14 14:55:37| at org.rzo.yajsw.app.WrapperJVMMain.executeMain(WrapperJVMMain.java:53)
FINEST|14443/5|10-12-14 14:55:37| at org.rzo.yajsw.app.WrapperJVMMain.main(WrapperJVMMain.java:36)
FINEST|14443/5|10-12-14 14:55:37|MyError:Socket Accepting has been caught in main loop.java.net.SocketException: Too many open files
INFO|wrapper|10-12-14 14:55:37|Trigger found: Too many open files
INFO|wrapper|10-12-14 14:55:37|too many restarts
FINEST|14443/5|10-12-14 14:55:37|java.net.SocketException: Too many open files
INFO|wrapper|10-12-14 14:55:37|Trigger found: Too many open files
INFO|wrapper|10-12-14 14:55:37|too many restarts
FINEST|14443/5|10-12-14 14:55:37| at java.net.PlainSocketImpl.socketAccept(Native Method)
FINEST|14443/5|10-12-14 14:55:37| at java.net.PlainSocketImpl.accept(PlainSocketImpl.java:390)
FINEST|14443/5|10-12-14 14:55:37| at java.net.ServerSocket.implAccept(ServerSocket.java:453)
FINEST|14443/5|10-12-14 14:55:37| at java.net.ServerSocket.accept(ServerSocket.java:421)
FINEST|14443/5|10-12-14 14:55:37| at commServer.main(commServer.java:29)
FINEST|14443/5|10-12-14 14:55:37| at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
FINEST|14443/5|10-12-14 14:55:37| at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
FINEST|14443/5|10-12-14 14:55:37| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
FINEST|14443/5|10-12-14 14:55:37| at java.lang.reflect.Method.invoke(Method.java:597)
FINEST|14443/5|10-12-14 14:55:37| at org.rzo.yajsw.app.WrapperJVMMain.executeMain(WrapperJVMMain.java:53)
FINEST|14443/5|10-12-14 14:55:37| at org.rzo.yajsw.app.WrapperJVMMain.main(WrapperJVMMain.java:36)
FINEST|14443/5|10-12-14 14:55:37|MyError:Socket Accepting has been caught in main loop.java.net.SocketException: Too many open files
INFO|wrapper|10-12-14 14:55:37|Trigger found: Too many open files
INFO|wrapper|10-12-14 14:55:37|too many restarts
FINEST|14443/5|10-12-14 14:55:37|java.net.SocketException: Too many open files
INFO|wrapper|10-12-14 14:55:37|Trigger found: Too many open files
INFO|wrapper|10-12-14 14:55:37|too many restarts
FINEST|14443/5|10-12-14 14:55:37| at java.net.PlainSocketImpl.socketAccept(Native Method)
FINEST|14443/5|10-12-14 14:55:37| at java.net.PlainSocketImpl.accept(PlainSocketImpl.java:390)
FINEST|14443/5|10-12-14 14:55:37| at java.net.ServerSocket.implAccept(ServerSocket.java:453)
FINEST|14443/5|10-12-14 14:55:37| at java.net.ServerSocket.accept(ServerSocket.java:421)
FINEST|14443/5|10-12-14 14:55:37| at commServer.main(commServer.java:29)
FINEST|14443/5|10-12-14 14:55:37| at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
FINEST|14443/5|10-12-14 14:55:37| at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
FINEST|14443/5|10-12-14 14:55:37| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
FINEST|14443/5|10-12-14 14:55:37| at java.lang.reflect.Method.invoke(Method.java:597)
FINEST|14443/5|10-12-14 14:55:37| at org.rzo.yajsw.app.WrapperJVMMain.executeMain(WrapperJVMMain.java:53)
FINEST|14443/5|10-12-14 14:55:37| at org.rzo.yajsw.app.WrapperJVMMain.main(WrapperJVMMain.java:36)
FINEST|14443/5|10-12-14 14:55:37|MyError:Socket Accepting has been caught in main loop.java.net.SocketException: Too many open files
INFO|wrapper|10-12-14 14:55:37|Trigger found: Too many open files
INFO|wrapper|10-12-14 14:55:37|too many restarts
FINEST|14443/5|10-12-14 14:55:37|java.net.SocketException: Too many open files
INFO|wrapper|10-12-14 14:55:37|Trigger found: Too many open files
INFO|wrapper|10-12-14 14:55:37|too many restarts
FINEST|14443/5|10-12-14 14:55:37| at java.net.PlainSocketImpl.socketAccept(Native Method)
FINEST|14443/5|10-12-14 14:55:37| at java.net.PlainSocketImpl.accept(PlainSocketImpl.java:390)
FINEST|14443/5|10-12-14 14:55:37| at java.net.ServerSocket.implAccept(ServerSocket.java:453)
FINEST|14443/5|10-12-14 14:55:37| at java.net.ServerSocket.accept(ServerSocket.java:421)
FINEST|14443/5|10-12-14 14:55:37| at commServer.main(commServer.java:29)
FINEST|14443/5|10-12-14 14:55:37| at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
FINEST|14443/5|10-12-14 14:55:37| at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
FINEST|14443/5|10-12-14 14:55:37| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
FINEST|14443/5|10-12-14 14:55:37| at java.lang.reflect.Method.invoke(Method.java:597)
FINEST|14443/5|10-12-14 14:55:37| at org.rzo.yajsw.app.WrapperJVMMain.executeMain(WrapperJVMMain.java:53)
FINEST|14443/5|10-12-14 14:55:37| at org.rzo.yajsw.app.WrapperJVMMain.main(WrapperJVMMain.java:36)
FINEST|14443/5|10-12-14 14:55:37|MyError:Socket Accepting has been caught in main loop.java.net.SocketException: Too many open files
INFO|wrapper|10-12-14 14:55:37|Trigger found: Too many open files
INFO|wrapper|10-12-14 14:55:37|too many restarts
FINEST|14443/5|10-12-14 14:55:37|java.net.SocketException: Too many open files
INFO|wrapper|10-12-14 14:55:37|Trigger found: Too many open files
INFO|wrapper|10-12-14 14:55:37|too many restarts
FINEST|14443/5|10-12-14 14:55:37| at java.net.PlainSocketImpl.socketAccept(Native Method)
FINEST|14443/5|10-12-14 14:55:37| at java.net.PlainSocketImpl.accept(PlainSocketImpl.java:390)
FINEST|14443/5|10-12-14 14:55:37| at java.net.ServerSocket.implAccept(ServerSocket.java:453)
FINEST|14443/5|10-12-14 14:55:37| at java.net.ServerSocket.accept(ServerSocket.java:421)
FINEST|14443/5|10-12-14 14:55:37| at commServer.main(commServer.java:29)
FINEST|14443/5|10-12-14 14:55:37| at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
INFO|14443/5|10-12-14 14:55:37|gobler execption OUTPUT 7155 Stream closed
INFO|14443/5|10-12-14 14:55:37|gobler terminated OUTPUT 7155
INFO|wrapper|10-12-14 15:00:34|removed pid file /var/run/wrapper.commServer.pid
INFO|wrapper|10-12-14 15:00:34|removed pid file /var/run/wrapper.commServer.pid
INFO|wrapper|10-12-14 15:00:34|removed pid file /var/run/wrapper.commServer.pid
INFO|wrapper|10-12-14 15:00:34|removed pid file /var/run/wrapper.commServer.pid
INFO|wrapper|10-12-14 15:01:02|created pid file /var/run/wrapper.commServer.pid
INFO|wrapper|10-12-14 15:01:02|set state IDLE->STARTING
INFO|wrapper|10-12-14 15:01:02|starting Process
INFO|wrapper|10-12-14 15:01:02|Controller State: UNKNOWN -> WAITING
INFO|wrapper|10-12-14 15:01:02|working dir /usr/local
INFO|wrapper|10-12-14 15:01:02|error initializing script
INFO|wrapper|10-12-14 15:01:02|exec:/usr/java/jdk1.6.0_18/bin/java -classpath /usr/local/yajsw-beta-10.2/./wrapperApp.jar:/usr/local -Xrs -Dwrapper.service=true -Dwrapper.console.visible=false -Dwrapper.visible=false -Dwrapper.pidfile=/var/run/wrapper.commServer.pid -Dwrapper.config=/usr/local/yajsw-beta-10.2/conf/wrapper.conf -Dwrapper.port=15003 -Dwrapper.key=-4075318938446295883 -Dwrapper.teeName=-4075318938446295883$1292310062486 -Dwrapper.tmpPath=/tmp org.rzo.yajsw.app.WrapperJVMMain
INFO|wrapper|10-12-14 15:01:03|started process 14658
INFO|wrapper|10-12-14 15:01:03|started process with pid 14658
INFO|wrapper|10-12-14 15:01:03|set state STARTING->RUNNING
INFO|14658/0|10-12-14 15:01:03|Dec 14, 2010 3:01:03 PM org.apache.commons.vfs.VfsLog info
INFO|14658/0|10-12-14 15:01:03|INFO: Using "/tmp/vfs_cache" as temporary files store.
INFO|14658/0|10-12-14 15:01:03|external stop false
INFO|14658/0|10-12-14 15:01:03|Dec 14, 2010 3:01:03 PM org.rzo.yajsw.app.WrapperManagerImpl start
INFO|14658/0|10-12-14 15:01:03|INFO: Logging ON
INFO|14658/0|10-12-14 15:01:03|Dec 14, 2010 3:01:03 PM org.rzo.yajsw.app.WrapperManagerImpl reconnect
INFO|14658/0|10-12-14 15:01:03|INFO: connecting to port 15003
INFO|14658/0|10-12-14 15:01:03|Controller State: WAITING -> ESTABLISHED
INFO|14658/0|10-12-14 15:01:03|Controller State: ESTABLISHED -> LOGGED_ON
INFO|14658/0|10-12-14 15:01:03|Dec 14, 2010 3:01:03 PM org.rzo.yajsw.nettyutils.LoggingFilter log
INFO|14658/0|10-12-14 15:01:03|INFO: [app][id: 0x26132ae7] OPEN
INFO|14658/0|10-12-14 15:01:03|Dec 14, 2010 3:01:03 PM org.rzo.yajsw.nettyutils.LoggingFilter log
INFO|14658/0|10-12-14 15:01:03|INFO: [app][id: 0x26132ae7] CONNECT: /127.0.0.1:15003

Basically it means that the process has reached the limit for its open file descriptors and the server can no longer accept connections from the client.

So basically now it boils down to answering the following questions:
- How much is the current limit (ulimit -n) per process? (you can also verify this by running a Java code which simply keeps on opening files and never closing it)
- How many simultaneous connections are you getting from the clients?
- Are you sure your code is written such that all resources are closed even when there is an exception?

Dear Sos,
1. When I run ulimit-n I get 1024. I dont get your when your run a java code keep opening and never closing it?
2. The problem I can give you a fix answer the simultaneous connections as the clients are gps devices normally they will send data the interval of 1 minute. Roughly now we have around 300 devices point to this server.
3. Are you saying for each of the exception I run this code w.close is it?

1. When I run ulimit-n I get 1024. I dont get your when your run a java code keep opening and never closing it?

1024 should be good enough given that you have just 300 clients. I suspect a resource leak here given that this problem of file handles occurs after running your application for a given amount of time.
Regarding the test code, what I meant was to write a simple code which recursively traverses directory for files, opens them and never closes the file handle. Just print out a counter after opening a file. After the exception "Too many open files" is thrown, just have a look at the counter which was printed and that would give you the number of max file handles your process is allowed to create.

3. Are you saying for each of the exception I run this code w.close is it?

No, what I'm trying to say that is make sure you close *all* resources eventually, exception or no exception.

I would recommend that you carry out a small experiment: deploy your server code. Monitor the current used file handles count by your process. Hit your application with a client request. Now monitor the used handle count. Ideally it should be the same as before the server started receiving requests. If the number of used file handles keeps on proportionally increasing with the number of requests, you have a leak.

Dear Sos,
Ok about the directory traverse I will go and try to run a code of this type. What type of resource is leaked? Is there any method to detect it?
1.I ran this command to find maximum open file descriptor "more /proc/sys/fs/file-max" is equal 1180130 (which I think is quite big rite).
2.This command "more /proc/sys/fs/file-nr" the results show me 2176 0 1180130 (This results never change I monitor for quite some time)
3. "lsof | wc -l" this show me a different result ranging from 3960 to 3970.

I am bit new in this monitor. I hope you can give me some guidance do you notice anything abnormal here?

Dear Sos,
Sorry I run again this command more /proc/sys/fs/file-nr. There is changes once is was 1196 0 1180130 then even another time it was 2304 0 1180130. In addition lsof | wc -l give a higher number in the range 3970 to 3990. Hope this gives a better indication now.

System administrators are pretty good at these monitoring things so if you can grab one to help you out, that would be the fastest route out. Using forum system for communication/debugging has a visible disconnect as you can see.

Anyways, AFAIK, more /proc/sys/fs/file-max gives you the *total* number of files handles allowed by the *OS*. It is more likely that you are hitting a per-process limit here ( ulimit -n ). Also, running just lsof | wc -l would be little use since it monitors the file handles for *all* processes visible to you. Find out the PID (process identifier) of your process using the ps ux command. Then do: lsof | grep YOUR_PID | wc -l . This would give you the number of file handles used by *your* Java process.

Regarding monitoring; you need to start the monitoring when your server process starts and look for changes when clients request services from your server. The number of used file handles *should* go up(at least by 1) when a client connects to your server. Like I mentioned in my previous post, ideally, the number of file handles free at the start of your server should be equal to the number of free file handles at any given point in time. If it isn't, then you are leaking resources.

Dear Sos,
Ok I did this command and below is the results. The problem I dont know which pid is for my java application is it 3073 or 3096.

ps -ef | grep java
root 3073 1 16 09:13 ? 00:00:02 /usr/java/jdk1.6.0_18/bin/java -Dwrapper.pidfile=/var/run/wrapper.commServer.pid -Dwrapper.service=true -Dwrapper.visible=false -jar /usr/local/yajsw-beta-10.2/wrapper.jar -c /usr/local/yajsw-beta-10.2/conf/wrapper.conf
root 3096 3073 43 09:13 ? 00:00:05 /usr/java/jdk1.6.0_18/bin/java -classpath /usr/local/yajsw-beta-10.2/./wrapperApp.jar:/usr/local -Xrs -Dwrapper.service=true -Dwrapper.console.visible=false -Dwrapper.visible=false -Dwrapper.pidfile=/var/run/wrapper.commServer.pid -Dwrapper.config=/usr/local/yajsw-beta-10.2/conf/wrapper.conf -Dwrapper.port=15003 -Dwrapper.key=3412727317078880866 -Dwrapper.teeName=3412727317078880866$1292375606402 -Dwrapper.tmpPath=/tmp org.rzo.yajsw.app.WrapperJVMMain
root 3196 2905 0 09:13 pts/0 00:00:00 grep java

Additionally I ran this command netstat -nat |grep 9000 | awk '{print $6}' | sort | uniq -c | sort -n and what I notice just before the server crash the number of time_wait keep increating. Then by browsing the log file I notice there quite a number of this error. Can it be that this error is also one of the cause for this problem? Must I close the connection in this catch

FINEST|24190/0|10-12-14 20:12:33|MyError:IOException has been caught in in the main first try
FINEST|24190/0|10-12-14 20:12:33|java.net.SocketException: Connection reset
FINEST|24190/0|10-12-14 20:12:33|	at java.net.SocketInputStream.read(SocketInputStream.java:168)
FINEST|24190/0|10-12-14 20:12:33|	at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:264)
FINEST|24190/0|10-12-14 20:12:33|	at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:306)
FINEST|24190/0|10-12-14 20:12:33|	at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:158)
FINEST|24190/0|10-12-14 20:12:33|	at java.io.InputStreamReader.read(InputStreamReader.java:167)
FINEST|24190/0|10-12-14 20:12:33|	at java.io.BufferedReader.fill(BufferedReader.java:136)
FINEST|24190/0|10-12-14 20:12:33|	at java.io.BufferedReader.read(BufferedReader.java:157)
FINEST|24190/0|10-12-14 20:12:33|	at ConnectionHandler.run(commServer.java:85)
FINEST|24190/0|10-12-14 20:12:33|	at java.lang.Thread.run(Thread.java:619)

Do you think I am lacking any close operation in my code? Thank you.

Ok I did this command and below is the results. The problem I dont know which pid is for my java application is it 3073 or 3096.

PID 3073 runs the JAR file wrapper.jar whereas PID 3096 runs the main class WrapperJVMMain. So the question is, is the code written by you packaged in the wrapper.jar file or wrapped up in the WrapperJVMMain class? That would be the answer to your question.

Regarding the stack trace, that probably means that the connection was severed/reset by the client and hence reading any more from the SocketInputStream is causing an exception.

Do you think I am lacking any close operation in my code?

Your code is a maze of if...else statements so its difficult to find out whether any connections/resources escape closure if an exception is thrown. There are now two things IMO which you can try out:
1) Run the code *without* the wrapper and see if it shows the same behaviour. Why do you require a wrapper anyway?
2) Remote debug your application. Make the necessary changes to the environment variables of your Java process, remote debug using Eclipse and you should be able to realize what resources aren't getting closed.

Dear Sos,
Actually I am also not too sure about the flow of the wrapper. I would guess that my application runs in the wrapper.jar then this application is run in the WrapperJVMMain class.
1. Why I run in wrapper is that I need the java to run as a service. So what is other solution to this any suggestion from you? The probem is that the server is in remote location so I am using putty to do all my operation.
2. What do you mean by remote debug?I am not too clear about how eclipse will help here.

Another question why at times there is not single time_wait but suddenly the time_wait grow and crash the applcation.Is there anything causing the time_wait to happen and grow? Thank you.

Thank you.

Actually I am also not too sure about the flow of the wrapper. I would guess that my application runs in the wrapper.jar then this application is run in the WrapperJVMMain class.

Then you need to grab someone who knows how wrapper works (the person who implemented this or maybe ask in the wrapper tool forums).

1. Why I run in wrapper is that I need the java to run as a service. So what is other solution to this any suggestion from you? The probem is that the server is in remote location so I am using putty to do all my operation.

The simplest possible option would be to spawn the process in background.

nohup java -cp .:somejar.jar -Dother=options SomeClasss &

2. What do you mean by remote debug?I am not too clear about how eclipse will help here.

You can use Eclipse to remote debug your application. So when ever a request comes, you can set a breakpoint in your server code and step through your entire code as it handles the client request. Eclipse also has the facility of attaching breakpoints for "specific" exceptions. Just add breakpoints for IOException and SocketException and you should be good to go. Search the net for tutorials for the same.

Another question why at times there is not single time_wait but suddenly the time_wait grow and crash the applcation.Is there anything causing the time_wait to happen and grow?

As per the man pages, TIME_WAIT is state is when the socket has been closed by is still waiting to handle packets in the network. Saying "suddenly the TIME_WAIT grow and crash" doesn't help here. We need exact numbers. After how many client requests? How many TIME_WAIT after starting the server? How many TIME_WAIT after handling a single client? Try to test things on a "per client" basis; that would have a higher chance of you finding out where the problem is and how many file descriptors are being leaked.

Dear Sos,
Ok I will ask again the wrapper guy.
1.Ok I will keep the nohup method as possible replacement the nohup can also write to log files rite. I will read up the net on that too first.
2.Eclipse I also will read the net first.
3. The exact number is 230 TIME_WAIT is when the system crash. The problem now I am monitoring with this command netstat -nat |grep 9000 | awk '{print $6}' | sort | uniq -c | sort -n. I notice when the established connection is below 200 no problem at all running for many hours with out problem. The moment I increase to say 216 then I get TIME_WAIT increase if not there is no TIME_WAIT at all shown in the command. So can this be due to the increase of clients.So must I increase the system file descriptor limit? Thank you.

3. The exact number is 230 TIME_WAIT is when the system crash. The problem now I am monitoring with this command netstat -nat |grep 9000 | awk '{print $6}' | sort | uniq -c | sort -n. I notice when the established connection is below 200 no problem at all running for many hours with out problem. The moment I increase to say 216 then I get TIME_WAIT increase if not there is no TIME_WAIT at all shown in the command. So can this be due to the increase of clients.So must I increase the system file descriptor limit? Thank you.

OK, let's go over this again. *How* are you exactly testing this thing? Are you running the stats on your existing deployment? If so, don't do that.

Do a fresh build of your code on some dev box or some other location for the purpose of testing. Start your server. Look at the number of file handles used by all "java" processes by doing: lsof | grep java | wc -l . Look at the number of TIME_WAIT now. Lets call these numbers collected as "set 1".

After this setup is done, create a client which hits your server and requests for some data. Only a single client. Now collect the same two statistics as mentioned above (set 1). It would obviously be more than the numbers collected previously. Now wait for some time, around 5/10 minutes and check the same two statistics again (set 3).

set 3 now should be the same as set 1. If it isn't, you have got a leak.

Dear Sos,
Yes I was runnning this stat on the same machine. Ok let me be open with you the problem is that we have over 300 gps devices set on the vehicles and running to set. To point to another IP it will take a very long process and to come back to another server it will be another long process which we can not quite afford due to time frame. So what is your other solution or work around? I dont understand how when you said create client and blast it ?

OK, my point was that deploy your application on a dev/test box and use a *dummy* client which simulates a GPS device. Surely you can do that, no? Basically it means running both applications side-by-side where one application would anyways be used by the devices as usual + the new application which you can use for testing. Don't you have a spare box running the same OS/configuration?

Also, the *simplest* hack here would be to just bump up the file descriptor limit and cross your fingers hoping that things work out. I guess the syntax is something along the lines of: ulimit -n 4096 but confirm this with your sys-admin. If there is a genuine code bug, you would get a "Too many files" error at some point in time, it's just that it would take a bit longer to reproduce it.

Dear Sos,
Ok at last I manage to create a dummy device using the telnet method. First I did wat on my existing box and another on new box but different java version.
Below is the lsof | grep java | wc -l values.

1.Original box
Connection with out sending data
a.With just the box listening(41)
b.With single client connection (42)
c.With the single client close connection it goes down to (41)
Connection with data being send
a.With just the box listening(41)
b.With single client connection (42)
c.When data sent(46)
d.With the single client close connection it goes down to (45)
Connection with data being send(multiple clients sending data)
a.With just the box listening(41)
b.With single client connection & more clients it will grow by 1(42,43 etc)
c.When data sent by each client grow by 1(46,47 etc)
d.When all the client have close connection it will drop to(45)

What suprises me is that when the data is sent it drop to 45 if not it drop to 41. Is it because of mysql connection and text file being operated. When data is send that means this part of code is operating while ((m=r.read()) != -1) then is written to the text file, update and insert database too.


2.Testing new box
Connection with out sending data
a.With just the box listening(31)
b.With single client connection (32)
c.With the single client close connection it goes down to (31)
Connection with data being send
a.With just the box listening(31)
b.With single client connection (32)
c.When data sent(36)
d.With the single client close connection it goes down to (35)
Connection with data being send(multiple clients sending data)
a.With just the box listening(31)
b.With single client connection & more clients it will grow by 1(32,33 etc)
c.When data sent by each client grow by 1(36,37 etc)
d.When all the client have close connection it will drop to(35)

What can you conclude from this fundamental test? The last value either 41 or 45 for the old box remain after 5 or 10 minutes. The above test results is purely running from java. Then I did with using the wrapper the results flow is same except your have to add 166 to the above values of original box. Thank you.

The increased number of file handles even after the entire client-server communication is done with is a bit troubling. That shouldn't be happening.

OK, now one more test. For the time being forget about the original box since we now have a dummy server deployment which we can use for our testing. Now, instead of printing out the *number* of file handles, print out the the entire data and redirect it to a file. Steps:
1) Start the server and don't request any data for the time being
2) Do lsof | grep java > before.txt .
3) Test for connection with data being sent; preferable would be to send data from your local box so that you don't end up using the file descriptors on the server box for testing your client.
4) After a single client transaction (send/receive) is complete, wait for some time (around 5 mins).
5) Now do lsof | grep java > after.txt .
6) Compare the two files; what do you see?

Edited 5 Years Ago by ~s.o.s~: n/a

Dear Sos,
Below is both contents of the files. The only difference I notice is that there is additional lines in the after.txt file that is

java 3610 root 8u IPv6 15008 TCP 192.168.2.15:webcache->63.92.48.60.kmr02-home.net.my:55006 (ESTABLISHED)
java 3610 root 10r REG 253,0 1163700 100364 /usr/java/jdk1.6.0_21/jre/lib/resources.jar
java 3610 root 12r CHR 1,8 1888 /dev/random
java 3610 root 13r CHR 1,9 770 /dev/urandom

Before

java 3610 root cwd DIR 253,0 4096 23987283 /root/Desktop/lsof
java 3610 root rtd DIR 253,0 4096 2 /
java 3610 root txt REG 253,0 47308 30574179 /usr/java/jdk1.6.0_21/bin/java
java 3610 root mem REG 253,0 125736 5604597 /lib/ld-2.5.so
java 3610 root mem REG 253,0 1611564 5604598 /lib/libc-2.5.so
java 3610 root mem REG 253,0 208352 5604605 /lib/libm-2.5.so
java 3610 root mem REG 253,0 16428 5604599 /lib/libdl-2.5.so
java 3610 root mem REG 253,0 129716 5604600 /lib/libpthread-2.5.so
java 3610 root mem REG 253,0 44060 5604601 /lib/librt-2.5.so
java 3610 root mem REG 253,0 101404 5604611 /lib/libnsl-2.5.so
java 3610 root mem REG 253,0 96117 100342 /usr/java/jdk1.6.0_21/jre/lib/i386/libnet.so
java 3610 root mem REG 253,0 56417984 29593755 /usr/lib/locale/locale-archive
java 3610 root mem REG 253,0 52079342 100367 /usr/java/jdk1.6.0_21/jre/lib/rt.jar
java 3610 root mem REG 253,0 117867 131660 /usr/java/jdk1.6.0_21/jre/lib/ext/comm.jar
java 3610 root mem REG 253,0 732695 100368 /usr/java/jdk1.6.0_21/jre/lib/ext/mysql-connector-java-5.1.12-bin.jar
java 3610 root mem REG 253,0 75985 100351 /usr/java/jdk1.6.0_21/jre/lib/i386/libzip.so
java 3610 root mem REG 253,0 46680 5603368 /lib/libnss_files-2.5.so
java 3610 root mem REG 253,0 32768 19759159 /tmp/hsperfdata_root/3610
java 3610 root mem REG 253,0 188954 100327 /usr/java/jdk1.6.0_21/jre/lib/i386/libjava.so
java 3610 root mem REG 253,0 56701 100350 /usr/java/jdk1.6.0_21/jre/lib/i386/libverify.so
java 3610 root mem REG 253,0 9679468 131094 /usr/java/jdk1.6.0_21/jre/lib/i386/server/libjvm.so
java 3610 root mem REG 253,0 38142 131086 /usr/java/jdk1.6.0_21/jre/lib/i386/jli/libjli.so
java 3610 root mem REG 253,0 38694 131090 /usr/java/jdk1.6.0_21/jre/lib/i386/native_threads/libhpi.so
java 3610 root 0u CHR 136,1 3 /dev/pts/1
java 3610 root 1u CHR 136,1 3 /dev/pts/1
java 3610 root 2u CHR 136,1 3 /dev/pts/1
java 3610 root 3r REG 253,0 52079342 100367 /usr/java/jdk1.6.0_21/jre/lib/rt.jar
java 3610 root 4r REG 253,0 732695 100368 /usr/java/jdk1.6.0_21/jre/lib/ext/mysql-connector-java-5.1.12-bin.jar
java 3610 root 5r REG 253,0 117867 131660 /usr/java/jdk1.6.0_21/jre/lib/ext/comm.jar
java 3610 root 6u IPv6 15007 TCP *:webcache (LISTEN)
java 3610 root 7u sock 0,5 15005 can't identify protocol

After

java 3610 root cwd DIR 253,0 4096 23987283 /root/Desktop/lsof
java 3610 root rtd DIR 253,0 4096 2 /
java 3610 root txt REG 253,0 47308 30574179 /usr/java/jdk1.6.0_21/bin/java
java 3610 root mem REG 253,0 125736 5604597 /lib/ld-2.5.so
java 3610 root mem REG 253,0 1611564 5604598 /lib/libc-2.5.so
java 3610 root mem REG 253,0 208352 5604605 /lib/libm-2.5.so
java 3610 root mem REG 253,0 16428 5604599 /lib/libdl-2.5.so
java 3610 root mem REG 253,0 129716 5604600 /lib/libpthread-2.5.so
java 3610 root mem REG 253,0 44060 5604601 /lib/librt-2.5.so
java 3610 root mem REG 253,0 101404 5604611 /lib/libnsl-2.5.so
java 3610 root mem REG 253,0 1163700 100364 /usr/java/jdk1.6.0_21/jre/lib/resources.jar
java 3610 root mem REG 253,0 96117 100342 /usr/java/jdk1.6.0_21/jre/lib/i386/libnet.so
java 3610 root mem REG 253,0 56417984 29593755 /usr/lib/locale/locale-archive
java 3610 root mem REG 253,0 52079342 100367 /usr/java/jdk1.6.0_21/jre/lib/rt.jar
java 3610 root mem REG 253,0 117867 131660 /usr/java/jdk1.6.0_21/jre/lib/ext/comm.jar
java 3610 root mem REG 253,0 732695 100368 /usr/java/jdk1.6.0_21/jre/lib/ext/mysql-connector-java-5.1.12-bin.jar
java 3610 root mem REG 253,0 75985 100351 /usr/java/jdk1.6.0_21/jre/lib/i386/libzip.so
java 3610 root mem REG 253,0 46680 5603368 /lib/libnss_files-2.5.so
java 3610 root mem REG 253,0 32768 19759159 /tmp/hsperfdata_root/3610
java 3610 root mem REG 253,0 188954 100327 /usr/java/jdk1.6.0_21/jre/lib/i386/libjava.so
java 3610 root mem REG 253,0 56701 100350 /usr/java/jdk1.6.0_21/jre/lib/i386/libverify.so
java 3610 root mem REG 253,0 9679468 131094 /usr/java/jdk1.6.0_21/jre/lib/i386/server/libjvm.so
java 3610 root mem REG 253,0 38142 131086 /usr/java/jdk1.6.0_21/jre/lib/i386/jli/libjli.so
java 3610 root mem REG 253,0 38694 131090 /usr/java/jdk1.6.0_21/jre/lib/i386/native_threads/libhpi.so
java 3610 root 0u CHR 136,1 3 /dev/pts/1
java 3610 root 1u CHR 136,1 3 /dev/pts/1
java 3610 root 2u CHR 136,1 3 /dev/pts/1
java 3610 root 3r REG 253,0 52079342 100367 /usr/java/jdk1.6.0_21/jre/lib/rt.jar
java 3610 root 4r REG 253,0 732695 100368 /usr/java/jdk1.6.0_21/jre/lib/ext/mysql-connector-java-5.1.12-bin.jar
java 3610 root 5r REG 253,0 117867 131660 /usr/java/jdk1.6.0_21/jre/lib/ext/comm.jar
java 3610 root 6u IPv6 15007 TCP *:webcache (LISTEN)
java 3610 root 7u sock 0,5 15005 can't identify protocol
java 3610 root 8u IPv6 15008 TCP 192.168.2.15:webcache->63.92.48.60.kmr02-home.net.my:55006 (ESTABLISHED)
java 3610 root 10r REG 253,0 1163700 100364 /usr/java/jdk1.6.0_21/jre/lib/resources.jar
java 3610 root 12r CHR 1,8 1888 /dev/random
java 3610 root 13r CHR 1,9 770 /dev/urandom

This question has already been answered. Start a new discussion instead.