diff options
Diffstat (limited to 'Demo/scripts')
| -rwxr-xr-x | Demo/scripts/newslist.doc | 59 | ||||
| -rwxr-xr-x | Demo/scripts/newslist.py | 309 |
2 files changed, 368 insertions, 0 deletions
diff --git a/Demo/scripts/newslist.doc b/Demo/scripts/newslist.doc new file mode 100755 index 0000000..87fd9ba --- /dev/null +++ b/Demo/scripts/newslist.doc @@ -0,0 +1,59 @@ + NEWSLIST + ======== + A program to assist HTTP browsing of newsgroups + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +WWW browsers such as NCSA Mosaic allow the user to read newsgroup +articles by specifying the group name in a URL eg 'news:comp.answers'. + +To browse through many groups, though, (and there are several thousand +of them) you really need a page or pages containing links to all the +groups. There are some good ones out there, for example, + + http://info.cern.ch/hypertext/DataSources/News/Groups/Overview.html + +is the standard one at CERN, but it only shows the groups available there, +which may be rather different from those available on your machine. + +Newslist is a program which creates a hierarchy of pages for you based +on the groups available from YOUR server. It is written in python - a +splendid interpreted object-oriented language which I suggest you get +right now from the directory /pub/python at ftp.cwi.nl, if you haven't +already got it. + +You should be able to see some sample output by looking at: + http://pelican.cl.cam.ac.uk/newspage/root.html + +Descriptions of newsgroups can be added from a file with one group +per line. eg: + + alt.foo Articles about foo + comp.bar Programming in 'bar' and related languages + +A suitable list detailing most groups can be found at ftp.uu.net in +/uunet-info/newsgroups.gz. + +Make sure you read the information at the beginning of the program source and +configure the variables before running. + +In addition to python, you need: + + An NNTP-based news feed. + A directory in which to put the pages. + +The programming is not very beautiful, but it works! It comes with no +warranty, express or implied, but with the hope that some others may +find it useful. + +Comments, improvements & suggestions welcomed. +Quentin Stafford-Fraser + + ---------------------------------------------------------------------- + Quentin Stafford-Fraser + http://pelican.cl.cam.ac.uk/people/qs101/me.html + + Cambridge University Computer Lab Rank Xerox Cambridge EuroPARC + qs101@cl.cam.ac.uk fraser@europarc.xerox.com + Tel: +44 223 334411 Tel: +44 223 341521 + Fax: +44 223 334679 Fax: +44 223 341510 + ---------------------------------------------------------------------- diff --git a/Demo/scripts/newslist.py b/Demo/scripts/newslist.py new file mode 100755 index 0000000..b0ac134 --- /dev/null +++ b/Demo/scripts/newslist.py @@ -0,0 +1,309 @@ +#! /usr/local/bin/python +####################################################################### +# Newslist $Revision$ +# +# Syntax: +# newslist [ -a ] +# +# This is a program to create a directory full of HTML pages +# which between them contain links to all the newsgroups available +# on your server. +# +# The -a option causes a complete list of all groups to be read from +# the server rather than just the ones which have appeared since last +# execution. This recreates the local list from scratch. Use this on +# the first invocation of the program. +# +# This assumes an NNTP news feed. +# +# Feel free to copy, distribute and modify this code for +# non-commercial use. If you make any useful modifications, let me +# know! +# +# (c) Quentin Stafford-Fraser 1994 +# fraser@europarc.xerox.com qs101@cl.cam.ac.uk +# # +####################################################################### +import sys,nntplib, string, marshal, time, os, posix, string + +####################################################################### +# Check these variables before running! # + +# Top directory. +# Filenames which don't start with / are taken as being relative to this. +##topdir='/anfs/qsbigdisc/web/html/newspage' +topdir = '/hosts/buizerd/ufs/www/cwi/cwionly/newstree' + +# The name of your NNTP host +# eg. +# newshost = 'nntp-serv.cam.ac.uk' +# or use following to get the name from the NNTPSERVER environment +# variable: +##newshost = posix.environ['NNTPSERVER'] +newshost = 'charon.cwi.nl' + +# The filename for a local cache of the newsgroup list +treefile = 'grouptree' + +# The filename for descriptions of newsgroups +# I found a suitable one at ftp.uu.net in /uunet-info/newgroups.gz +# You can set this to '' if you don't wish to use one. +##descfile = 'newsgroups' +descfile = '/usr/lib/news/newsgroups' + +# The directory in which HTML pages should be created +# eg. +# pagedir = '/usr/local/lib/html/newspage' +# pagedir = 'pages' +pagedir = topdir + +# The html prefix which will refer to this directory +# eg. +# httppref = '/newspage/', +# or leave blank for relative links +# between pages. (Recommended) +httppref = '' + +# The name of the 'root' news page in this directory. +# A .html suffix will be added. +##rootpage = 'root' +rootpage = 'index' + +# Set skipempty to 0 if you wish to see links to empty groups as well. +# Only affects the -a option. +##skipempty = 1 +skipempty = 0 + +# --------------------------------------------------------------------- +# Less important personal preferences: + +# Sublistsize controls the maximum number of items the will appear as +# an indented sub-list before the whole thing is moved onto a different +# page. The smaller this is, the more pages you will have, but the +# shorter each will be. +sublistsize = 4 + +# That should be all. # +####################################################################### + +from nntplib import NNTP +from stat import * + +rcsrev = '$Revision$'[11:15] +desc = {} + +# Make (possibly) relative filenames into absolute ones +treefile = os.path.join(topdir,treefile) +descfile = os.path.join(topdir,descfile) +page = os.path.join(topdir,pagedir) + +# First the bits for creating trees --------------------------- + +# Addtotree creates/augments a tree from a list of group names +def addtotree(tree, groups): + print 'Updating tree...' + for i in groups: + parts = string.splitfields(i,'.') + makeleaf(tree, parts) + +# Makeleaf makes a leaf and the branch leading to it if necessary +def makeleaf(tree,path): + j = path[0] + l = len(path) + + if not tree.has_key(j): + tree[j] = {} + if l == 1: + tree[j]['.'] = '.' + if l > 1: + makeleaf(tree[j],path[1:]) + +# Then the bits for outputting trees as pages ---------------- + +# Createpage creates an HTML file named <root>.html containing links +# to those groups beginning with <root>. + +def createpage(root, tree, p): + filename = os.path.join(pagedir,root+'.html') + if root == rootpage: + detail = '' + else: + detail = ' under ' + root + f = open(filename,'w') + # f.write('Content-Type: text/html\n') + f.write('<TITLE>Newsgroups available' + detail + '</TITLE>\n') + f.write('<H1>Newsgroups available' + detail +'</H1>\n') + f.write('<A HREF="'+httppref+rootpage+'.html">Back to top level</A><P>\n') + printtree(f,tree,0,p) + f.write('<I>This page automatically created by \'newslist\' v. '+rcsrev+'.') + f.write(time.ctime(time.time()) + '</I><P>') + f.close() + +# Printtree prints the groups as a bulleted list. Groups with +# more than <sublistsize> subgroups will be put on a separate page. +# Other sets of subgroups are just indented. + +def printtree(f, tree, indent, p): + global desc + l = len(tree) + + if l > sublistsize and indent>0: + # Create a new page and a link to it + f.write('<LI><B><A HREF="'+httppref+p[1:]+'.html">') + f.write(p[1:]+'.* ...') + f.write('</A></B>\n') + createpage(p[1:], tree, p) + return + + kl = tree.keys() + + if l > 1: + kl.sort() + if indent > 0: + # Create a sub-list + f.write('<LI><B>'+p[1:]+'</B>\n<UL>') + else: + # Create a main list + f.write('<UL>') + indent = indent + 1 + + for i in kl: + if i == '.': + # Output a newsgroup + f.write('<LI><A HREF="news:' + p[1:] + '">'+ p[1:] + '</A> ') + if desc.has_key(p[1:]): + f.write(' <I>'+desc[p[1:]]+'</I>\n') + else: + f.write('\n') + else: + # Output a hierarchy + printtree(f,tree[i], indent, p+'.'+i) + + if l > 1: + f.write('\n</UL>') + +# Reading descriptions file --------------------------------------- + +# This returns an array mapping group name to its description + +def readdesc(): + global desc + + desc = {} + + if descfile == '': + return + + try: + d = open(descfile, 'r') + print 'Reading descriptions...' + except (IOError): + print 'Failed to open description file ' + descfile + return + l = d.readline() + while l != '': + bits = string.split(l) + try: + grp = bits[0] + dsc = string.join(bits[1:]) + if len(dsc)>1: + desc[grp] = dsc + except (IndexError): + pass + l = d.readline() + +# Now the main program -------------------------------------------- + +def main(): + global desc + + connected = 0 + tree={} + + # Check that the output directory exists + if not os.path.isdir(pagedir): + print 'Directory '+pagedir+' does not exist.' + print 'Shall I create it for you? (y/n)' + if sys.stdin.readline()[0] == 'y': + try: + os.mkdir(pagedir,0777) + except: + print 'Sorry - failed!' + sys.exit(1) + else: + print 'OK. Exiting.' + sys.exit(1) + + try: + print 'Connecting to '+newshost+'...' + if sys.version[0] == '0': + s = NNTP.init(newshost) + else: + s = NNTP(newshost) + connected = 1 + except (nntplib.error_temp, nntplib.error_perm), x: + print 'Error connecting to host:' + print x + print 'I\'ll try to use just the local list.' + + # If -a is specified, read the full list of groups from server + if connected and len(sys.argv) > 1 and sys.argv[1] == '-a': + print 'Getting list of all groups...' + treedate='010101' + info = s.list()[1] + groups = [] + print 'Processing...' + if skipempty: + print '\nIgnoring following empty groups:' + for i in info: + if skipempty and string.atoi(i[1]) < string.atoi(i[2]): + print i[0]+' ', + else: + groups.append(i[0]) + print '\n(End of empty groups)' + + # Otherwise just read groups created since local file last modified. + else: + print 'Reading current local group list...' + try: + treetime = time.localtime(os.stat(treefile)[ST_MTIME]) + except: + print '\n*** Failed to open local group cache '+treefile + print 'If this is the first time you have run newslist, then' + print 'use the -a option to create it.' + sys.exit(1) + treedate = '%02d%02d%02d' % (treetime[0] % 100 ,treetime[1], treetime[2]) + try: + dump = open(treefile,'r') + tree = marshal.load(dump) + dump.close() + except (IOError): + print 'Cannot open local group list ' + treefile + + if connected: + print 'Getting list of new groups since start of '+treedate+'...', + groups = s.newgroups(treedate,'000001')[1] + print 'got '+`len(groups)`+'.' + + if connected: + addtotree(tree, groups) + try: + dump = open(treefile,'w') + groups = marshal.dump(tree,dump) + dump.close() + print 'Saved list to '+treefile+'\n' + except: + print 'Sorry - failed to write to local group cache '+treefile + print 'Does it (or its directory) have the correct permissions?' + sys.exit(1) + + readdesc() + print 'Creating pages...' + createpage(rootpage, tree, '') + print 'Done' + + +main() + +# That's all folks +###################################################################### |