Merging more additions to the module code.

bookwormDB/CreateDatabase.py

@@ -35,11 +35,17 @@ def text_id_dbm():
     """
     dbm = anydbm.open(".bookworm/texts/textids.dbm","c")
     for file in os.listdir(".bookworm/texts/textids"):
-        for line in open(".bookworm/texts/textids/" + file):        
+        for line in open(".bookworm/texts/textids/" + file):
             line = line.rstrip("\n")
             splat = line.split("\t")
-            dbm[splat[1]] = splat[0]
-
+            try:
+                dbm[splat[1]] = splat[0]
+            except IndexError:
+                if line=="":
+                    # It's OK to have a blank line, let's say.
+                    continue
+                else:
+                    raise
 class DB:
     def __init__(self,dbname=None):
         config = ConfigParser.ConfigParser(allow_no_value=True)

bookwormDB/MetaParser.py

@@ -3,6 +3,7 @@
 import dateutil.parser
 import json
 import sys
+import logging
 
 fields_to_derive = []
 fields = []
@@ -174,8 +175,13 @@ def ParseJSONCatalog(target="default",source = "default"):
                                 time = int(inttime/7)*7
                                 #Not starting on Sunday or anything funky like that. Actually, I don't know what we're starting on. Adding an integer here would fix that.
                                 line[k] = time
+                            elif derive['resolution'] == 'day':
+                                k = "%s_day" % field['field']
+                                dt = date(intent[0], intent[1], intent[2])
+                                inttime = DaysSinceZero(dt)
+                                line[k] = inttime
                             else:
-                                sys.stderr.write('Resolution currently not supported.\n')
+                                logging.warning('Resolution %s currently not supported.' %(derive['resolution']))
                                 continue
                     except ValueError:
                         # One of out a million Times articles threw this with

bookwormDB/manager.py

@@ -26,6 +26,14 @@ class BookwormManager(object):
     def __init__(self,cnf_file="bookworm.cnf",database=None,user=None,password=None):
         # This will likely be changed if it isn't None.
         import ConfigParser
+
+        self.basedir = None
+        for i in range(10):
+            basedir = "../"*i
+            if os.path.exists(basedir + ".bookworm"):
+                self.basedir = basedir
+            if self.basedir==None:
+                logging.debug("No bookworm directory found; proceeding on nonetheless.")
 
         self.dbname=database
 
@@ -200,9 +208,9 @@ def extension(self,args):
         Creates (or updates) an extension
         """
 
-        if not os.path.exists("extensions"):
-            os.makedirs("extensions")
-        my_extension = Extension(args)
+        if not os.path.exists(self.basedir + ".bookworm/extensions"):
+            os.makedirs(self.basedir + ".bookworm/extensions")
+        my_extension = Extension(args,basedir = self.basedir)
         my_extension.clone_or_pull()
         my_extension.make()
 
@@ -412,9 +420,9 @@ class Extension(object):
     they are build using `make`.
     """
 
-    def __init__(self,args):
+    def __init__(self,args,basedir="./"):
         self.args = args
-        self.dir = "extensions/" + re.sub(".*/","",self.args.url)
+        self.dir = basedir + ".bookworm/extensions/" + re.sub(".*/","",self.args.url)
 
     def clone_or_pull(self):
         if not os.path.exists(self.dir):

bookwormDB/variableSet.py

@@ -60,8 +60,12 @@ def __init__(self):
         for filelist in filelists:
             for line in open(".bookworm/texts/textids/%s" % filelist):
                 parts = line.replace('\n', '').split("\t")
-                self[parts[1]] = int(parts[0])
-                numbers.append(int(parts[0]))
+                if len(parts)==2:
+                    # Allowing terminal newline. 
+                    self[parts[1]] = int(parts[0])
+                    numbers.append(int(parts[0]))
+
+
         self.new = open('.bookworm/texts/textids/new', 'a')
         self.max = max(numbers)
 

test_bookworm/testAPI.py

@@ -21,7 +21,7 @@ def test_server_connection(self):
     which isn't strictly necessary for a bookworm to be built.
     """
 
-    def mtest_bookworm_creation(self):
+    def test_bookworm_creation(self):
         """
         Creates a test bookworm. Removes any existing databases called "federalist_bookworm"
         """
@@ -110,7 +110,8 @@ class Dummy:
             field_descriptions = None # Test the guessing at field_descriptions while we're at it
         import os
         os.chdir("/tmp/federalist/federalist-bookworm-master")
-        manager.add_metadata(Dummy)            
+        manager.add_metadata(Dummy)
+        self.assertTrue(1==1)
 
     def test_metadata_addition_can_be_retrieved(self):
         from bookwormDB.general_API import SQLAPIcall as SQLAPIcall
@@ -121,13 +122,16 @@ def test_metadata_addition_can_be_retrieved(self):
         query = {
                 "database":"federalist_bookworm",
                 "search_limits":{},
-                "counttype":"TextPercent",
+                "counttype":"TextCount",
                 "groups":["oddness"],
                 "method":"return_json"
         }
 
         m = json.loads(SQLAPIcall(query).execute())
+        # Even or odd is one of two things.
         self.assertTrue(len(m)==2)
+        # Since the first paragraph is even, there should be more of those.
+        self.assertTrue(m['odd'][0]>=m['even'][0])