Merge pull request #491 from OpenFn/cursor-helper

Cursor Helper

.changeset/odd-apples-wait.md

@@ -0,0 +1,5 @@
+---
+'@openfn/language-common': minor
+---
+
+Added cursor() helper

packages/common/ast.json

@@ -1312,6 +1312,83 @@
       },
       "valid": true
     },
+    {
+      "name": "cursor",
+      "params": [
+        "value",
+        "options"
+      ],
+      "docs": {
+        "description": "Sets a cursor property on state.\nSupports natural language dates like `now`, `today`, `yesterday`, `n hours ago`, `n days ago`, and `start`,\nwhich will be converted relative to the environment (ie, the Lightning or CLI locale). Custom timezones \nare not yet supported.\nSee the usage guide at @{link https://docs.openfn.org/documentation/jobs/job-writing-guide#using-cursors}",
+        "tags": [
+          {
+            "title": "public",
+            "description": null,
+            "type": null
+          },
+          {
+            "title": "example",
+            "description": "cursor($.cursor, { defaultValue: 'today' })",
+            "caption": "Use a cursor from state if present, or else use the default value"
+          },
+          {
+            "title": "example",
+            "description": "cursor(22)",
+            "caption": "Use a pagination cursor"
+          },
+          {
+            "title": "function",
+            "description": null,
+            "name": null
+          },
+          {
+            "title": "param",
+            "description": "the cursor value. Usually an ISO date, natural language date, or page number",
+            "type": {
+              "type": "NameExpression",
+              "name": "any"
+            },
+            "name": "value"
+          },
+          {
+            "title": "param",
+            "description": "options to control the cursor.",
+            "type": {
+              "type": "NameExpression",
+              "name": "object"
+            },
+            "name": "options"
+          },
+          {
+            "title": "param",
+            "description": "set the cursor key. Will persist through the whole run.",
+            "type": {
+              "type": "NameExpression",
+              "name": "string"
+            },
+            "name": "options.key"
+          },
+          {
+            "title": "param",
+            "description": "the value to use if value is falsy",
+            "type": {
+              "type": "NameExpression",
+              "name": "any"
+            },
+            "name": "options.defaultValue"
+          },
+          {
+            "title": "returns",
+            "description": null,
+            "type": {
+              "type": "NameExpression",
+              "name": "Operation"
+            }
+          }
+        ]
+      },
+      "valid": false
+    },
     {
       "name": "map",
       "params": [

packages/common/src/Adaptor.js

@@ -6,8 +6,9 @@ import { parse } from 'csv-parse';
 import { Readable } from 'node:stream';
 
 import { request } from 'undici';
+import { format } from 'date-fns';
 
-import { expandReferences as newExpandReferences } from './util';
+import { expandReferences as newExpandReferences, parseDate } from './util';
 
 export * as beta from './beta';
 export * as http from './http.deprecated';
@@ -777,3 +778,60 @@ export function validate(schema = 'schema', data = 'data') {
     }
   };
 }
+
+let cursorStart = undefined;
+let cursorKey = 'cursor';
+
+/**
+ * Sets a cursor property on state.
+ * Supports natural language dates like `now`, `today`, `yesterday`, `n hours ago`, `n days ago`, and `start`,
+ * which will be converted relative to the environment (ie, the Lightning or CLI locale). Custom timezones 
+ * are not yet supported.
+ * See the usage guide at @{link https://docs.openfn.org/documentation/jobs/job-writing-guide#using-cursors}
+ * @public
+ * @example <caption>Use a cursor from state if present, or else use the default value</caption>
+ * cursor($.cursor, { defaultValue: 'today' })
+ * @example <caption>Use a pagination cursor</caption>
+ * cursor(22)
+ * @function
+ * @param {any} value - the cursor value. Usually an ISO date, natural language date, or page number
+ * @param {object} options - options to control the cursor.
+ * @param {string} options.key - set the cursor key. Will persist through the whole run.
+ * @param {any} options.defaultValue - the value to use if value is falsy
+ * @returns {Operation}
+ */
+export function cursor(value, options = {}) {
+  return (state) => {
+    const [resolvedValue, resolvedOptions] = newExpandReferences(state, value, options);
+
+    const {
+      defaultValue, // if there is no cursor on state, this will be used
+      key, // the key to use on state
+    } = resolvedOptions;
+
+    if (key) {
+      cursorKey = key;
+    }
+
+    if (!cursorStart) {
+      cursorStart = new Date();
+    }
+
+    const cursor = resolvedValue ?? defaultValue;
+    if (typeof cursor === 'string') {
+      const date = parseDate(cursor, cursorStart)
+      if (date instanceof Date && date.toString !== "Invalid Date") {
+        state[cursorKey] = date.toISOString();
+        // Log the converted date in a very international, human-friendly format
+        // See https://date-fns.org/v3.6.0/docs/format
+        const formatted = format(date, 'HH:MM d MMM yyyy (OOO)')
+        console.log(`Setting cursor "${cursor}" to: ${formatted}`);
+        return state;
+      }
+    }
+    state[cursorKey] = cursor;
+    console.log('Setting cursor to:', cursor);
+
+    return state;
+  }
+}

packages/common/src/util/index.js

@@ -1,2 +1,5 @@
 export * from './http';
 export * from './references';
+import parseDate from './parse-date';
+
+export  { parseDate }

packages/common/src/util/parse-date.js

@@ -0,0 +1,34 @@
+import { startOfToday, startOfYesterday, subHours, subDays, startOfDay } from 'date-fns'
+
+// Helper function to parse a natural-language date string into an ISO date
+export default (d, startDate) => {
+  try {
+    if (d === 'start') {
+      return startDate;
+    } else if (d === 'now' || d === 'end') {
+      return new Date()
+    }
+    else if (d === 'today') {
+      return startOfToday()
+    }
+    else if (d === 'yesterday') {
+      return startOfYesterday()
+    }
+    else if (/(hours? ago)$/.test(d)) {
+      // return the same minute n hours ago
+      const [diff] = d.match(/\d+/)
+      return subHours(new Date(), diff)
+    }
+    else if (/(days? ago)$/.test(d)) {
+      // return the start of today - n days
+      const [diff] = d.match(/\d+/)
+      return startOfDay(subDays(new Date(), diff))
+    }
+  } catch(e) {
+    console.log(`Error converting ${d} into a date`)
+    console.log(e)
+  }
+
+  // Just return the value if we couldn't parse it
+  return d;
+}

packages/common/test/index.test.js

@@ -7,6 +7,7 @@ import {
   arrayToString,
   chunk,
   combine,
+  cursor,
   dataPath,
   dataValue,
   each,
@@ -29,6 +30,7 @@ import {
   toArray,
   validate,
 } from '../src/Adaptor';
+import { startOfToday } from 'date-fns';
 
 const mockAgent = new MockAgent();
 setGlobalDispatcher(mockAgent);
@@ -861,3 +863,73 @@ describe('validate', () => {
     expect(result.validationErrors).to.eql([]);
   });
 });
+
+describe('cursor', () => {
+  it('should set a cursor on state', () => {
+    const state = {}
+    const result = cursor(1234)(state)
+    expect(result.cursor).to.eql(1234);
+  });
+
+  it('should set a cursorStart on state', () => {
+    const state = {}
+    const date = new Date();
+    const result = cursor('start')(state)
+    const resultDate = new Date(result.cursor)
+    expect(resultDate.toDateString()).to.eql(date.toDateString())
+  });
+
+  it('should set a cursor on state with a natural language timestamp', () => {
+    const state = {}
+
+    const date = startOfToday().toISOString()
+    const result = cursor('today')(state)
+    expect(result.cursor).to.eql(date);
+  });
+
+  it('should not blow up if an arbitrary string is passed', () => {
+    const state = {}
+
+    const str = 'rock the cashbah'
+    const result = cursor(str)(state)
+    expect(result.cursor).to.eql(str);
+  });
+
+  it('should clear the cursor', () => {
+    const state = {
+      cursor: new Date()
+    }
+    const result = cursor()(state)
+    expect(result.cursor).to.eql(undefined)
+  });
+
+  it('should use a default value', () => {
+    const state = {}
+    const result = cursor(state.cursor, { defaultValue: 33 })(state)
+    expect(result.cursor).to.eql(33)
+  });
+
+  it('should use a custom key', () => {
+    const state = {}
+    const result = cursor(44, { key: 'page' })(state)
+    expect(result.page).to.eql(44)
+  });
+
+  it('should re-use a custom key', () => {
+    const state = {}
+    const result1 = cursor(44, { key: 'page' })(state)
+    expect(result1.page).to.eql(44)
+
+    const result2 = cursor(55)(state)
+    expect(result2.page).to.eql(55)
+  });
+
+  // testing the log output is hard here, I've only verified it manally
+  it('should use an object', () => {
+    const state = {}
+    const c = { page: 22, next: 23 }
+    const result = cursor(c, { key: 'cursor' })(state)
+    expect(result.cursor).to.eql(c)
+  });
+
+});