Downloading thread code.

[Gribouillis]

The following program is able to download the python programs contained in a thread of the python forum. Just start the program, it will prompt you for the thread number and create a directory with the code extracted from the thread. I used it to download all the wx examples. Note that the algorithm is very primitive: only the code written in language tags is extracted (raw code is not), and data attached to posts is not downloaded. Also it's not robust, if the formatting in daniweb changes tomorrow, it won't work anymore :)

#!/usr/bin/env python
# danidown.py
from htmllib import HTMLParser 
from formatter import AbstractFormatter ,AbstractWriter ,NullWriter 
import re 
from os import mkdir 
from os .path import isdir ,join as pjoin 
from urllib2 import urlopen 

class aThread (object ):
  itsCntPattern =re .compile (r"Page \d+ of (\d+)")
  def __init__ (o ,theThreadNumber ):
    o .itsNumber =theThreadNumber 
    o ._itsPageCnt =None 
    o .itsReply =0 
    o .itsCodeIndex =0 
  def itsUrl (o ,thePage =1 ):
    x =""if (thePage ==1 )else "-%d"%thePage 
    return "http://www.daniweb.com/forums/thread%d%s.html"%(
    o .itsNumber ,x )
  def itsContent (o ,thePage =1 ):
    theUrl =o .itsUrl (thePage )
    f =urlopen (theUrl )
    s =f .read ()
    f .close ()
    return s 
  @property 
  def itsPageCnt (o ):
    if o ._itsPageCnt is None :
      o ._itsPageCnt =1 
      theContent =o .itsContent (1 )
      theMatch =o .itsCntPattern .search (theContent )
      if theMatch is not None :
        o ._itsPageCnt =int (theMatch .group (1 ))
    return o ._itsPageCnt 
  @property 
  def itsTriples (o ):
    theCnt =o .itsPageCnt 
    printMessage ("The thread contains %d pages..."%theCnt )
    for i in xrange (1 ,theCnt +1 ):
      printMessage ("Page %d..."%i )
      theWriter =Writer1 ()
      theParser =HTMLParser (AbstractFormatter (theWriter ))
      theContent =o .itsContent (i )
      theParser .feed (theContent )
      theParser .close ()
      for theTriple in theWriter .itsTriples :
        yield theTriple 

  @property 
  def itsFolder (o ):
    return "thread%d"%o .itsNumber 

  def itsReplyFolder (o ,n ):
    return pjoin (o .itsFolder ,"reply%d"%n )

  def doDownload (o ):
    for theReply ,theAuthor ,theCode in o .itsTriples :
      theFolder =o .itsReplyFolder (theReply )
      if theReply >o .itsReply :
        printMessage ("reply %d..."%theReply )
        if o .itsReply ==0 :
          if not isdir (o .itsFolder ):
            mkdir (o .itsFolder )
        o .itsReply =theReply 
        o .itsCodeIndex =0 
        if not isdir (theFolder ):
          mkdir (theFolder )
        f =open (pjoin (theFolder ,"author"),"w")
        f .write (theAuthor +"\n")
        f .close ()
      o .itsCodeIndex +=1 
      f =open (pjoin (theFolder ,"prog%d.py"%o .itsCodeIndex ),"w")
      f .write (theCode )
      f .close ()
    print "done."

class Writer1 (NullWriter ):
  def __init__ (o ):
    NullWriter .__init__ (o )
    o .isInCode =False 
    o .itsCode =None 
    o .itsAuthor ="unknown"
    o .itsAnswer =0 
    o .justReadAuthor =False 
    o .nextIsAuthor =False 
    o .nextIsNumber =False 
    o .itsTriples =[]
  def send_label_data (o ,data ):
#print "send_label_data(%s)" % repr(data)
    if o .isInCode :
      assert (data [-1 ]==".")
      n =int (data [:-1 ])
      o .itsCode .append ([])
      assert (len (o .itsCode )==n )
    elif data =="1.":
      o .isInCode =True 
      o .itsCode =[[]]
  def send_literal_data (o ,data ):
#print "send_literal_data(%s)" % repr(data)
    if o .isInCode and data !="\xa0":
      o .itsCode [-1 ].append (data )
  def send_line_break (o ):
    if o .isInCode :
      o .itsCode [-1 ]="".join (o .itsCode [-1 ])
  def send_paragraph (o ,data ):
#print "send_paragraph(%s)" % repr(data)
    if o .isInCode :
      theCode ="\n".join (o .itsCode )
      o .itsCode =None 
      o .isInCode =False 
      o .itsTriples .append ((o .itsAnswer ,o .itsAuthor ,theCode ))
  def send_flowing_data (o ,data ):
    if o .nextIsNumber :
      o .itsAnswer =int (data )
      o .nextIsNumber =False 
    elif o .justReadAuthor :
      if data ==" #":
        o .nextIsNumber =True 
        o .justReadAuthor =False 
    elif o .nextIsAuthor :
      o .itsAuthor =data 
      o .justReadAuthor =True 
      o .nextIsAuthor =False 
    elif data .startswith (" Solved Threads:"):
      o .nextIsAuthor =True 

def printMessage (msg ):
  print msg 

if __name__ =="__main__":
  n =int (raw_input ("Enter thread number: "))
  theThread =aThread (n )
  theThread .doDownload ()