Merge pull request #38 from adelosa/feature/remove-mongo

Remove support for mongo extract function.

HISTORY.rst

@@ -6,6 +6,7 @@ History
 0.5.0 (2016-10-03)
 ------------------
 * Fixed version display in release version.
+* Removed support for mongo extract
 
 0.4.8 (2016-10-02)
 ------------------

docs/installation.rst

@@ -4,9 +4,7 @@ Installation
 
 At the command line::
 
-    $ easy_install mciutil
-
-Or, if you have virtualenvwrapper installed::
-
-    $ mkvirtualenv mciutil
+    $ virtualenv mciutil
+    $ source ./mciutil/bin/activate
     $ pip install mciutil
+

docs/usage.rst

@@ -13,7 +13,7 @@ paramconv - Convert MPE parameter extract files
 -----------------------------------------------
 Use this tool if you are moving between a mainframe and PC based clearing
 application and need the parameter files available on both platforms.
-The app currently works exclusively with 1014 blocked file format which is
+The app currently works with 1014 blocked file and VBS format which is
 used for communications between MasterCard and the member.
 
 Most simple usage, just provide a MasterCard MPE file to convert::
@@ -22,14 +22,18 @@ Most simple usage, just provide a MasterCard MPE file to convert::
 
 This runs with the following assumptions
 
-* Input file format is EBCDIC
-* Output file is the input file plus a '.out' extension
+* Input file format is EBCDIC with 1014 blocking
+* Output file name is the input file name plus a '.out' extension
 
-If you have a ASCII file and want to convert it to EBCDIC, you need to provide
+If you have an ASCII file and want to convert it to EBCDIC, you need to provide
 the source format type::
 
     paramconv -s ascii <inputfile>
 
+If you are working with VBS format, just add the --no1014blocking flag to the command::
+
+    paramconv -s ascii --no1014blocking <inputfile>
+
 You can change the output file location::
 
     paramconv -o <outputfile> <inputfile>
@@ -50,7 +54,7 @@ Extract command
 ^^^^^^^^^^^^^^^
 Use this command to extract transactions from a MasterCard
 IPM format file into usable formats like csv. The app currently works
-exclusively with 1014 blocked file format which is used for communications
+with 1014 blocked file format and VBS format which is used for communications
 between MasterCard and the member.
 
 Get a csv file from an IPM file::
@@ -59,30 +63,25 @@ Get a csv file from an IPM file::
 
 This runs with the following assumptions
 
-* Input file format is EBCDIC
-* Output file is the input file plus a '.csv' extension
+* Input file is EBCDIC and 1014 blocked format
+* Output file name is the input file name plus a '.csv' extension
 
 .. attention::
    Currently python 2.6 does not print a header row as the csv library only
    added this support in python 2.7. This function may be added in a future
    release.
 
-If you need to process an ASCII encoded file::
+If you need to process an ASCII encoded 1014 blocked file::
 
     mideu extract -s ascii <filename>
 
-You can change the output file location for the CSV file::
-
-    mideu extract <inputfile> --csvoutputfile <outputfile>
+If you are working with VBS format, just add the --no1014blocking flag to the command::
 
-You can also load the transactions into a MongoDB collection::
+    mideu extract -s ascii --no1014blocking <filename>
 
-    mideu extract <inputfile> --mongohost localhost --mongodb testdb
+You can change the CSV output file name and location::
 
-The transactions will be added to a collection called ``mastercardtransactions``
-Currently the existing collection is deleted prior to the load.
-You should consider this functionality to be beta and subject to change in the
-future. Feel free to suggest changes.
+    mideu extract <inputfile> --csvoutputfile <outputfile>
 
 To get all the usage details::
 
@@ -100,14 +99,18 @@ Most simple usage, just provide a MasterCard IPM file to convert::
 
 This runs with the following assumptions
 
-* Input file format is EBCDIC
-* Output file is the input file plus a '.out' extension
+* Input file is EBCDIC and 1014 blocked format
+* Output file name is the input file name plus a '.out' extension
 
 If you have a ASCII file and want to convert it to EBCDIC, you need to provide
 the source format type::
 
     mideu convert -s ascii <inputfile>
 
+If you are working with VBS format, just add the --no1014blocking flag to the command::
+
+    mideu convert -s ascii --no1014blocking <filename>
+
 To get all the usage details::
 
     mideu convert --help
@@ -153,19 +156,10 @@ Structure::
             field_length: 0
             field_processor: PAN
 
-**mongo_config**
-    Specify mongo host and port details. Command line options will override
-    options provided in a config file
-
-Structure::
-
-    mongo_config:
-        host: 192.168.99.100:27017
-        db: test
-
-
+MasterCard file formats
+-----------------------
 VBS file format
----------------
+^^^^^^^^^^^^^^^
 *added 0.4.6*
 
 This format is a basic variable record format.
@@ -190,7 +184,7 @@ with the file finishing with a zero length record length::
     00000040: 42 42 42 31 32 33 00 00  00 00                    BBB123....
 
 1014 blocked file format
-------------------------
+^^^^^^^^^^^^^^^^^^^^^^^^
 This is the default format used by mciutil
 
 This is the same as VBS format with 1014 blocking applied.

mciutil/cli/extract.py

@@ -8,13 +8,11 @@
 
 import logging
 import yaml
-import pymongo
 
 from mciutil import unblock, vbs_unpack, get_message_elements
 from mciutil.cli.common import (
     get_config_filename,
     add_to_csv,
-    filter_data_list,
 )
 
 LOGGER = logging.getLogger(__name__)
@@ -57,49 +55,13 @@ def extract_command(args):
 
     print("\nCompleted processing {0} records".format(len(input_file)))
 
-    # write to output
-    if args.mongo or args.mongohost or args.mongodb:
-        mongo_config = config["mongo_config"]
-        if args.mongohost:
-            mongo_config["host"] = args.mongohost
-        if args.mongodb:
-            mongo_config["db"] = args.mongodb
-
-        add_to_mongo(
-            output_list,
-            config['output_data_elements'],
-            mongo_config)
-
-    else:  # default write to csv
-        if args.csvoutputfile:
-            csv_output_filename = args.csvoutputfile
-        else:
-            csv_output_filename = args.input + ".csv"
-        add_to_csv(
-            output_list,
-            config['output_data_elements'],
-            csv_output_filename
-        )
-
-
-def add_to_mongo(data_list, field_list, mongo_config):
-    """
-    Loads data into a mongo db collection
-
-    :param data_list: list of dictionaries that contain the data to be loaded
-    :param field_list: list of fields in the dictionary to be loaded
-    :param mongo_config: config dictionary required to connect to mongo
-    :return: result from mongo insert
-    """
-    LOGGER.info("Connecting to mongo at %s", mongo_config["host"])
-    client = pymongo.MongoClient("mongodb://" + mongo_config["host"])
-    db_client = client[mongo_config["db"]]
-
-    LOGGER.info("Deleting existing items")
-    db_client.mastercardtransactions.delete_many({})
-
-    LOGGER.info("Filtering %s items", len(data_list))
-    filtered_data_list = filter_data_list(data_list, field_list)
-
-    LOGGER.info("Loading %s items", len(filtered_data_list))
-    return db_client.mastercardtransactions.insert_many(filtered_data_list)
+    # write to csv - utf-8 encoded
+    if args.csvoutputfile:
+        csv_output_filename = args.csvoutputfile
+    else:
+        csv_output_filename = args.input + ".csv"
+    add_to_csv(
+        output_list,
+        config['output_data_elements'],
+        csv_output_filename
+    )

mciutil/cli/mideu.py

@@ -104,13 +104,6 @@ def _add_extract_args(parser):
     :param parser: the argparse parser
     :return: None
     """
-    mongo_arg_group = parser.add_argument_group("mongo output options")
-    mongo_arg_group.add_argument("--mongo",
-                                 help="add to mongo",
-                                 action="store_true")
-    mongo_arg_group.add_argument("--mongohost", help="mongo hostname")
-    mongo_arg_group.add_argument("--mongodb", help="mongo db")
-
     csv_arg_group = parser.add_argument_group("csv output options")
     csv_arg_group.add_argument("--csvoutputfile", help="Output filename")
 

mciutil/cli/mideu.yml

@@ -1,8 +1,3 @@
-# Specify mongodb connection details
-mongo_config:
-  host: 192.168.99.100:27017
-  db: test
-
 # Provides specifications for ISO8583 fields
 bit_config:
   1:

requirements.txt

@@ -3,5 +3,4 @@ wheel==0.23.0
 hexdump>=3.2
 bitarray>=0.8.1
 PyYAML>=3.10
-pymongo>=3.0.3
 argparse>=1.4.0

setup.py

@@ -15,7 +15,7 @@
     history = history_file.read().replace('.. :changelog:', '')
 
 requirements = [
-    'PyYAML', 'hexdump', 'bitarray', 'pymongo', 'argparse'
+    'PyYAML', 'hexdump', 'bitarray', 'argparse'
 ]
 
 test_requirements = [

tests/test_mideu.py

@@ -136,11 +136,6 @@ def test_output_byte_field(self):
         self.assertEqual(field.decode(), 'this is some text')
 
 
-class MongoOutputTest(TestCase):
-    def test_mongo_output(self):
-        pass
-
-
 class FilteredDictionaryTest(TestCase):
     def test_filter_dict(self):
         dict = {"a": b("123"), "b": b("456"), "c": b("789")}