cut_down_a_file_size.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# file splitter\n",
        "this takes an excell file and splits it into 3000 line peices for URL update.\n",
        "saves out as tab-delimited text files.\n"
      ],
      "metadata": {
        "id": "Mg2mwdAxt0Dy"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8a7qivKrtha5",
        "outputId": "88360b85-7dc0-4d9c-ad3e-55d14f901056"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Splitting completed. Tab-delimited text files are in the '/content/drive/MyDrive/Colab Notebooks/Crossref_notebooks' directory.\n"
          ]
        }
      ],
      "source": [
        "import pandas as pd\n",
        "import os\n",
        "import sys\n",
        "### this splits an .xlsx file ###\n",
        "\n",
        "\n",
        "def split_excel_to_text(input_excel, output_directory, lines_per_file=3000):\n",
        "    # Read the Excel file into a Pandas DataFrame\n",
        "    df = pd.read_excel(input_excel)\n",
        "\n",
        "    # Create output directory if it doesn't exist\n",
        "    os.makedirs(output_directory, exist_ok=True)\n",
        "\n",
        "    # Split DataFrame into smaller DataFrames with a maximum number of lines\n",
        "    split_dfs = [df.iloc[i:i + lines_per_file] for i in range(0, len(df), lines_per_file)]\n",
        "\n",
        "    # Save each smaller DataFrame to a separate tab-delimited text file\n",
        "    for i, split_df in enumerate(split_dfs):\n",
        "        output_file = os.path.join(output_directory, f'output_{i + 1}.txt')\n",
        "        split_df.to_csv(output_file, sep='\\t', index=False)\n",
        "\n",
        "    print(f\"Splitting completed. Tab-delimited text files are in the '{output_directory}' directory.\")\n",
        "\n",
        "# Input Excel file\n",
        "\n",
        "input_excel = \"/content/drive/MyDrive/Colab Notebooks/Crossref_notebooks/DOIs,_URLs_for_prefix_10.3920.xlsx\"\n",
        " # Output directory\n",
        "\n",
        "output_dir = \"/content/drive/MyDrive/Colab Notebooks/Crossref_notebooks\"\n",
        "\n",
        "# Call the function\n",
        "split_excel_to_text(input_excel, output_dir)\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "### this one reads a .txt file instead ###\n",
        "\n",
        "\n",
        "def split_excel_to_text(input_excel, output_directory, lines_per_file=3000):\n",
        "    # Read the text file into a Pandas DataFrame\n",
        "\n",
        "    df = pd.read_csv(input_text, sep='\\t')\n",
        "\n",
        "    # Create output directory if it doesn't exist\n",
        "    os.makedirs(output_directory, exist_ok=True)\n",
        "\n",
        "    # Split DataFrame into smaller DataFrames with a maximum number of lines\n",
        "    split_dfs = [df.iloc[i:i + lines_per_file] for i in range(0, len(df), lines_per_file)]\n",
        "\n",
        "    # Save each smaller DataFrame to a separate tab-delimited text file\n",
        "    for i, split_df in enumerate(split_dfs):\n",
        "        output_file = os.path.join(output_directory, f'output_{i + 1}.txt')\n",
        "        split_df.to_csv(output_file, sep='\\t', index=False)\n",
        "\n",
        "    print(f\"Splitting completed. Tab-delimited text files are in the '{output_directory}' directory.\")\n",
        "\n",
        "# Input Excel file\n",
        "\n",
        "input_text = \"/content/drive/MyDrive/Colab Notebooks/Crossref_notebooks/DOIs,_URLs_for_prefix_10.3920.xls\"\n",
        " # Output directory\n",
        "\n",
        "output_dir = \"/content/drive/MyDrive/Colab Notebooks/Crossref_notebooks\"\n",
        "\n",
        "# Call the function\n",
        "split_excel_to_text(input_text, output_dir)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Yx_ebmhHyFET",
        "outputId": "0dac5bd1-b18f-492e-8afa-2335924edda8"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Splitting completed. Tab-delimited text files are in the '/content/drive/MyDrive/Colab Notebooks/Crossref_notebooks' directory.\n"
          ]
        }
      ]
    }
  ]
}