diff --git a/learning_observer/learning_observer/blob_storage.py b/learning_observer/learning_observer/blob_storage.py new file mode 100644 index 000000000..f86c996e6 --- /dev/null +++ b/learning_observer/learning_observer/blob_storage.py @@ -0,0 +1,36 @@ +import learning_observer.kvs +import learning_observer.stream_analytics.helpers as sa_helpers + +def state_blob(): + '''Dummy function for the reducer name portion of the + KVS key + ''' + pass + +def _make_key(user_id, source, activity): + '''Helper function to format keys for the KVS + ''' + key = sa_helpers.make_key( + state_blob, + { + sa_helpers.EventField('source'): source, + sa_helpers.EventField('activity'): activity, + sa_helpers.KeyField.STUDENT: user_id + }, + sa_helpers.KeyStateType.INTERNAL + ) + return key + +async def fetch_blob(user_id, source, activity): + '''Fetch the blob from the KVS + ''' + key = _make_key(user_id, source, activity) + kvs = learning_observer.kvs.KVS() + return await kvs[key] + +async def save_blob(user_id, source, activity, blob): + '''Store a blob in the KVS + ''' + key = _make_key(user_id, source, activity) + kvs = learning_observer.kvs.KVS() + await kvs.set(key, blob) diff --git a/learning_observer/learning_observer/incoming_student_event.py b/learning_observer/learning_observer/incoming_student_event.py index 6cb7de904..b8a996b25 100644 --- a/learning_observer/learning_observer/incoming_student_event.py +++ b/learning_observer/learning_observer/incoming_student_event.py @@ -38,6 +38,7 @@ import learning_observer.auth.events import learning_observer.adapters.adapter import learning_observer.blacklist +import learning_observer.blob_storage import learning_observer.constants as constants @@ -427,7 +428,7 @@ async def decode_lock_fields(events): ''' async for event in events: if event['event'] == 'lock_fields': - if event['fields'].get('source', '') != lock_fields.get('source', ''): + if 'source' not in event['fields'] or event['fields'].get('source', '') != lock_fields.get('source', ''): lock_fields.update(event['fields']) else: event.update(lock_fields) @@ -447,6 +448,35 @@ async def filter_blacklist_events(events): await ws.send_json(bl_status) await ws.close() + async def process_blob_storage_events(events): + '''HACK This function manages events related to storing and + retrieving blobs from server-side storage. It is primarily + used for LO Assess. Ideally, this functionality should reside + in an independent module, rather than being directly integrated + into Learning Observer, as it is currently implemented. + ''' + async for event in events: + # Extract metadata + if event['event'] in ['save_blob', 'fetch_blob']: + user_id = event['auth']['user_id'] + source = event['source'] + activity = event['activity'] + + # Save, fetch, or ignore (continue) + if event['event'] == 'save_blob': + await learning_observer.blob_storage.save_blob( + user_id, source, activity, + event['blob'] + ) + elif event['event'] == 'fetch_blob': + blob = await learning_observer.blob_storage.fetch_blob(user_id, source, activity) + await ws.send_json({ + 'status': 'fetch_blob', + 'data': blob + }) + else: + yield event + async def check_for_reducer_update(events): '''Check to see if the reducers updated ''' @@ -470,6 +500,7 @@ async def process_ws_message_through_pipeline(): events = decode_lock_fields(events) events = handle_auth_events(events) events = filter_blacklist_events(events) + events = process_blob_storage_events(events) events = check_for_reducer_update(events) events = pass_through_reducers(events) # empty loop to start the generator pipeline diff --git a/modules/lo_event/lo_event/browserStorage.js b/modules/lo_event/lo_event/browserStorage.js index 613d0fb31..181eeb5b0 100644 --- a/modules/lo_event/lo_event/browserStorage.js +++ b/modules/lo_event/lo_event/browserStorage.js @@ -58,7 +58,7 @@ const thunkStorage = { * `storage.sync.get`/`chrome.sync.get` API. */ function getWithCallback (getItem) { - function get (items, callback) { + function get (items, callback = () => {}) { if (typeof items === 'string') { items = [items]; } @@ -77,7 +77,7 @@ function getWithCallback (getItem) { * `storage.sync.set`/`chrome.sync.set` API. */ function setWithCallback (setItem) { - function set (items, callback) { + function set (items, callback = () => {}) { for (const item in items) { setItem(item, items[item]); } diff --git a/modules/lo_event/lo_event/lo_assess/reducers.js b/modules/lo_event/lo_event/lo_assess/reducers.js index 7cb469ec1..2f4ef610e 100644 --- a/modules/lo_event/lo_event/lo_assess/reducers.js +++ b/modules/lo_event/lo_event/lo_assess/reducers.js @@ -5,11 +5,13 @@ const DEBUG = false; const dclog = (...args) => {if(DEBUG) {console.log.apply(console, Array.from(args));} }; export const LOAD_DATA_EVENT = 'LOAD_DATA_EVENT'; +export const LOAD_STATE = 'LOAD_STATE'; export const NAVIGATE = 'NAVIGATE'; export const SHOW_SECTION='SHOW_SECTION'; export const STEPTHROUGH_NEXT = 'STEPTHROUGH_NEXT'; export const STEPTHROUGH_PREV = 'STEPTHROUGH_PREV'; export const STORE_VARIABLE = 'STORE_VARIABLE'; +export const STORE_SETTING = 'STORE_SETTING'; export const UPDATE_INPUT = 'UPDATE_INPUT'; export const UPDATE_LLM_RESPONSE = 'UPDATE_LLM_RESPONSE'; export const VIDEO_TIME_EVENT = 'VIDEO_TIME_EVENT'; @@ -46,11 +48,14 @@ export const updateResponseReducer = (state = initialState, action) => { registerReducer( [LOAD_DATA_EVENT, + LOAD_STATE, NAVIGATE, SHOW_SECTION, STEPTHROUGH_NEXT, STEPTHROUGH_PREV, + STORE_SETTING, STORE_VARIABLE, UPDATE_INPUT, - UPDATE_LLM_RESPONSE, VIDEO_TIME_EVENT], + UPDATE_LLM_RESPONSE, + VIDEO_TIME_EVENT], updateResponseReducer ); diff --git a/modules/lo_event/lo_event/reduxLogger.js b/modules/lo_event/lo_event/reduxLogger.js index c5b74e60f..9dd53f33f 100644 --- a/modules/lo_event/lo_event/reduxLogger.js +++ b/modules/lo_event/lo_event/reduxLogger.js @@ -20,20 +20,92 @@ */ import * as redux from 'redux'; import { thunk } from 'redux-thunk'; +import { createStateSyncMiddleware, initMessageListener } from 'redux-state-sync'; +import debounce from 'lodash/debounce'; + +import * as util from './util.js'; const EMIT_EVENT = 'EMIT_EVENT'; const EMIT_LOCKFIELDS = 'EMIT_LOCKFIELDS'; const EMIT_SET_STATE = 'SET_STATE'; +let IS_LOADED = false; + // TODO: Import debugLog and use those functions. const DEBUG = false; -function debug_log(...args) { - if(DEBUG) { +function debug_log (...args) { + if (DEBUG) { console.log(...args); } } +/** + * Update the redux logger's state with `data`. + * This is fired when consuming a custom `fetch_blob` + * event. + */ +export function handleLoadState (data) { + IS_LOADED = true; + const state = store.getState(); + if (data) { + setState( + { + ...state, + ...data, + settings: { + ...state.settings, + reduxStoreStatus: IS_LOADED + } + }); + } else { + debug_log('No data provided while handling state from server, continuing.'); + setState( + { + ...state, + settings: { + ...state.settings, + reduxStoreStatus: IS_LOADED + } + }); + } +} + +async function saveStateToLocalStorage (state) { + if (!IS_LOADED) { + debug_log('Not saving store locally because IS_LOADED is set to false.'); + return; + } + + try { + const KEY = state?.settings?.reduxID || 'redux'; + const serializedState = JSON.stringify(state); + localStorage.setItem(KEY, serializedState); + } catch (e) { + // Ignore + } +} + +/** + * Dispatch a `save_blob` event on the redux + * logger. + */ +async function saveStateToServer (state) { + if (!IS_LOADED) { + debug_log('Not saving store on the server because IS_LOADED is set to false.'); + return; + } + + try { + // console.log("dispatching save_blob") + util.dispatchCustomEvent('save_blob', { detail: state }); + // store.dispatch('save_blob', { detail: state }); + } catch (e) { + // Ignore + debug_log('Error in dispatch', { e }); + } +} + // Action creator function This is a little bit messy, since we // duplicate type from the payload. It's not clear if this is a good // idea. We used to have `type` be set to the current contents of @@ -65,11 +137,11 @@ const emitSetState = (state) => { }; }; -function store_last_event_reducer(state = {}, action) { +function store_last_event_reducer (state = {}, action) { return { ...state, event: action.payload }; }; -function lock_fields_reducer(state = {}, action) { +function lock_fields_reducer (state = {}, action) { const payload = JSON.parse(action.payload); return { ...state, @@ -116,7 +188,7 @@ export const updateComponentStateReducer = ({}) => (state = initialState, action return new_state; } -function set_state_reducer(state = {}, action) { +function set_state_reducer (state = {}, action) { return action.payload; } @@ -126,16 +198,16 @@ const BASE_REDUCERS = { [EMIT_SET_STATE]: [set_state_reducer] } -const APPLICATION_REDUCERS = { -} +const APPLICATION_REDUCERS = {}; export const registerReducer = (keys, reducer) => { const reducerKeys = Array.isArray(keys) ? keys : [keys]; reducerKeys.forEach(key => { - if (!APPLICATION_REDUCERS[key]) + debug_log('registering key: ' + key); + if (!APPLICATION_REDUCERS[key]) { APPLICATION_REDUCERS[key] = []; - + } APPLICATION_REDUCERS[key].push(reducer); }); return reducer; @@ -145,11 +217,20 @@ export const registerReducer = (keys, reducer) => { const reducer = (state = {}, action) => { let payload; - debug_log("Reducing ", action," on ", state); + debug_log('Reducing ', action, ' on ', state); state = BASE_REDUCERS[action.redux_type] ? composeReducers(...BASE_REDUCERS[action.redux_type])(state, action) : state; if (action.redux_type === EMIT_EVENT) { payload = JSON.parse(action.payload); + if (action.type === 'save_setting') { + return { + ...state, + settings: { + ...state.settings, + payload + } + }; + } debug_log(Object.keys(payload)); if (APPLICATION_REDUCERS[payload.event]) { @@ -160,21 +241,24 @@ const reducer = (state = {}, action) => { return state; }; - const eventQueue = []; const composeEnhancers = (typeof window !== 'undefined' && window.__REDUX_DEVTOOLS_EXTENSION_COMPOSE__) || redux.compose; - // This should just be redux.applyMiddleware(thunk)) // There is a bug in our version of redux-thunk where, in node, this must be thunk.default. // // This shows up as an error in the test case. If the error goes away, we should switch this // back to thunk. +// const presistedState = loadState(); + export let store = redux.createStore( reducer, - {event: null}, // Base state - composeEnhancers(redux.applyMiddleware(thunk.default || thunk)) + { event: null }, // Base state + composeEnhancers(redux.applyMiddleware((thunk.default || thunk), createStateSyncMiddleware())) ); + +initMessageListener(store); + let promise = null; let previousEvent = null; let lockFields = null; @@ -200,13 +284,35 @@ function composeReducers(...reducers) { } export function setState(state) { - debug_log("Set state called"); + debug_log('Set state called'); + if (Object.keys(state).length === 0) { + const storeState = store.getState(); + state = { + settings: { + ...storeState.settings, + reduxStoreStatus: IS_LOADED + } + }; + } store.dispatch(emitSetState(state)); } +const debouncedSaveStateToLocalStorage = debounce((state) => { + saveStateToLocalStorage(state); +}, 1000); + +const debouncedSaveStateToServer = debounce((state) => { + saveStateToServer(state); +}, 1000); + function initializeStore () { store.subscribe(() => { const state = store.getState(); + // we use debounce to save the state once every second + // for better performances in case multiple changes occur in a short time + debouncedSaveStateToLocalStorage(state); + debouncedSaveStateToServer(state); + if (state.lock_fields) { lockFields = state.lock_fields.fields; } @@ -233,7 +339,7 @@ function initializeStore () { }); } -export function reduxLogger (subscribers, initialState = {}) { +export function reduxLogger (subscribers, initialState = null) { if (subscribers != null) { eventSubscribers = subscribers; } @@ -254,7 +360,9 @@ export function reduxLogger (subscribers, initialState = {}) { logEvent.getLockFields = function () { return lockFields; }; - setState(initialState); + // do we want to initialize the store here? We set it to the stored state in create store + // if (initialState) { + // } return logEvent; } @@ -282,3 +390,6 @@ export const awaitEvent = () => { promise.resolve = resolvePromise; return promise; }; + +// Start listening for fetch +util.consumeCustomEvent('fetch_blob', handleLoadState); diff --git a/modules/lo_event/lo_event/util.js b/modules/lo_event/lo_event/util.js index ee53636f5..0b1a3f940 100644 --- a/modules/lo_event/lo_event/util.js +++ b/modules/lo_event/lo_event/util.js @@ -410,19 +410,59 @@ export function formatTime(seconds) { * When working in a browser, we want to dispatch the event via the * `window` object. */ -export function dispatchCustomEvent(eventName, detail) { - const event = new CustomEvent(eventName, { detail }); - if (typeof window !== "undefined") { +export function dispatchCustomEvent (eventName, detail) { + const event = new CustomEvent(eventName, detail); + if (typeof window !== 'undefined') { // Web page: dispatch directly on window window.dispatchEvent(event); - } else if (typeof chrome !== "undefined" && chrome.runtime && chrome.runtime.sendMessage) { - // Chrome extension background script: use chrome.runtime to send messages - chrome.runtime.sendMessage({ eventName, detail }, (response) => { - if (chrome.runtime.lastError) { - console.warn(`No listeners found for event, ${eventName}, in this context.`); - } - }); + } else if (typeof chrome !== 'undefined' && chrome.runtime && chrome.runtime.sendMessage) { + // Chrome extension background script: use chrome.runtime to send messages + chrome.runtime.sendMessage({ eventName, detail }, (response) => { + if (chrome.runtime.lastError) { + console.warn(`No listeners found for event, ${eventName}, in this context.`); + } + }); + } else { + console.warn('Event dispatching is not supported in this environment.'); + } +} + +/** + * This function consumes a custom event in the appropriate context for + * our environment. + * + * When working in an extension, it listens for messages via the + * `chrome.runtime.onMessage` object. + * + * When working in a browser, it listens for events on the + * `window` object. + */ +export function consumeCustomEvent (eventName, callback) { + if (typeof window !== 'undefined') { + // Web page: listen for the event on the window object + const listener = (event) => { + callback(event.detail); + }; + window.addEventListener(eventName, listener); + + // Return a function to remove the event listener + return () => window.removeEventListener(eventName, listener); + } else if (typeof chrome !== 'undefined' && chrome.runtime && chrome.runtime.onMessage) { + // Chrome extension background script: listen for messages via chrome.runtime + const listener = (message, sender, sendResponse) => { + if (message.eventName === eventName) { + callback(message.detail, sender); + sendResponse?.({ status: 'received' }); + } + }; + chrome.runtime.onMessage.addListener(listener); + + // Return a function to remove the message listener + return () => chrome.runtime.onMessage.removeListener(listener); } else { - console.warn("Event dispatching is not supported in this environment."); + console.warn('Event consumption is not supported in this environment.'); + return () => { + console.warn('No listener to remove in this environment.'); + }; } } diff --git a/modules/lo_event/lo_event/websocketLogger.js b/modules/lo_event/lo_event/websocketLogger.js index 714ae1104..8724b72d7 100644 --- a/modules/lo_event/lo_event/websocketLogger.js +++ b/modules/lo_event/lo_event/websocketLogger.js @@ -125,7 +125,6 @@ export function websocketLogger (server = {}) { function receiveMessage (event) { const response = JSON.parse(event.data); - switch (response.status) { case 'blocklist': debug.info('Received block error from server'); @@ -136,17 +135,20 @@ export function websocketLogger (server = {}) { ); break; case 'auth': - storage.set({user_id: response.user_id}); - util.dispatchCustomEvent("auth", { detail: { user_id: response.user }}); + storage.set({ user_id: response.user_id }); + util.dispatchCustomEvent('auth', { detail: { user_id: response.user } }); break; // These should probably be behind a feature flag, as they assume // we trust the server. case 'local_storage': - storage.set({[response.key]: response.value}); + storage.set({ [response.key]: response.value }); break; case 'browser_event': util.dispatchCustomEvent(response.event_type, { detail: response.detail }); break; + case 'fetch_blob': + util.dispatchCustomEvent('fetch_blob', { detail: response.data }); + break; default: debug.info(`Received response we do not yet handle: ${response}`); break; @@ -184,5 +186,11 @@ export function websocketLogger (server = {}) { queue.enqueue(data); }; + function handleSaveBlob (blob) { + queue.enqueue(JSON.stringify({ event: 'save_blob', blob })); + } + + util.consumeCustomEvent('save_blob', handleSaveBlob) + return wsLogData; } diff --git a/modules/lo_event/package.json b/modules/lo_event/package.json index d62e93855..f5e85ee58 100644 --- a/modules/lo_event/package.json +++ b/modules/lo_event/package.json @@ -26,6 +26,7 @@ "type": "module", "dependencies": { "aws-sdk": "^2.1614.0", + "debounce": "^2.2.0", "http-server": "^14.1.1", "indexeddb-js": "^0.0.14", "jasmine": "^5.1.0", @@ -33,6 +34,7 @@ "react": "^18.3.1", "react-dom": "^18.3.1", "redux": "^5.0.1", + "redux-state-sync": "^3.1.4", "redux-thunk": "^3.1.0", "sqlite3": "^5.1.6", "uuid": "^10.0.0", diff --git a/modules/lo_toy_sba/MANIFEST.in b/modules/lo_toy_sba/MANIFEST.in new file mode 100644 index 000000000..e69de29bb diff --git a/modules/lo_toy_sba/README.md b/modules/lo_toy_sba/README.md new file mode 100644 index 000000000..377245633 --- /dev/null +++ b/modules/lo_toy_sba/README.md @@ -0,0 +1,8 @@ +# LO Toy SBA + +This module provides various functionality for using the Toy SBA code within the Learning Observer system. + +The included functionality + +1. Providing a stub reducer so the Toy SBA can save/fetch state - we need a reducer that matches the source of LO Event +1. Serve the Toy SBA built NextJS output - not yet implemented diff --git a/modules/lo_toy_sba/lo_toy_sba/__init__.py b/modules/lo_toy_sba/lo_toy_sba/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/modules/lo_toy_sba/lo_toy_sba/module.py b/modules/lo_toy_sba/lo_toy_sba/module.py new file mode 100644 index 000000000..6d26ffeaa --- /dev/null +++ b/modules/lo_toy_sba/lo_toy_sba/module.py @@ -0,0 +1,80 @@ +''' +Toy-SBA Module + +Toy-SBA Module +''' +import learning_observer.communication_protocol.util +from learning_observer.stream_analytics.helpers import KeyField, Scope + +import lo_toy_sba.reducers + +# Name for the module +NAME = 'Toy-SBA Module' + +''' +Define execution DAGs for this module. We provide a default DAG +for fetching information from the provided reducer. The internal +structure looks like: + +`execution_dag`: defined directed acyclic graph (DAG) for querying data + : q.select() # or some other communication protocol query +`exports`: fetchable nodes from the execution dag + : { + "returns": , + "parameters": ["list", "of", "parameters", "needed"] + } + +NOTE interfacing with the communication protocol may change, +the current flow is the first iteration. We will mark where things +ought to be improved. +''' +EXECUTION_DAG = learning_observer.communication_protocol.util.generate_base_dag_for_student_reducer('student_event_counter', 'lo_toy_sba') + +''' +This is a simple reducer we use to ensure events are +passed into the event pipeline to save/fetch state. +We need a reducer whose context matches the source of +a page using LO Event. +''' +REDUCERS = [ + { + 'context': 'org.ets.sba', + 'scope': Scope([KeyField.STUDENT]), + 'function': lo_toy_sba.reducers.student_event_counter, + 'default': {'count': 0} + } +] + +''' +Which pages to link on the home page. +''' +COURSE_DASHBOARDS = [ + # { + # 'name': NAME, + # 'url': "/lo_toy_sba/toy-sba/", + # "icon": { + # "type": "fas", + # "icon": "fa-play-circle" + # } + # } +] + + +''' +Additional API calls we can define, this one returns the colors of the rainbow +''' +EXTRA_VIEWS = [ + # { + # 'name': 'Colors of the Rainbow', + # 'suburl': 'api/llm', + # 'method': 'POST', + # 'handler': function_to_call + # } +] + +''' +Built NextJS pages we want to serve. +''' +NEXTJS_PAGES = [ + # {'path': 'toy_sba/'} +] diff --git a/modules/lo_toy_sba/lo_toy_sba/reducers.py b/modules/lo_toy_sba/lo_toy_sba/reducers.py new file mode 100644 index 000000000..f34958ac7 --- /dev/null +++ b/modules/lo_toy_sba/lo_toy_sba/reducers.py @@ -0,0 +1,16 @@ +''' +This file defines reducers we wish to add to the incoming event +pipeline. The `learning_observer.stream_analytics` package includes +helper functions for Scoping the and setting the null state. +''' +from learning_observer.stream_analytics.helpers import student_event_reducer + + +@student_event_reducer(null_state={"count": 0}) +async def student_event_counter(event, internal_state): + ''' + An example of a per-student event counter + ''' + state = {"count": internal_state.get('count', 0) + 1} + + return state, state diff --git a/modules/lo_toy_sba/pyproject.toml b/modules/lo_toy_sba/pyproject.toml new file mode 100644 index 000000000..8fe2f47af --- /dev/null +++ b/modules/lo_toy_sba/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools>=42", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/modules/lo_toy_sba/setup.cfg b/modules/lo_toy_sba/setup.cfg new file mode 100644 index 000000000..9c06f7fa1 --- /dev/null +++ b/modules/lo_toy_sba/setup.cfg @@ -0,0 +1,10 @@ +[metadata] +name = Toy-SBA Module +description = Module for serving the Toy SBA work. + +[options] +packages = lo_toy_sba + +[options.entry_points] +lo_modules = + lo_toy_sba = lo_toy_sba.module diff --git a/package.json b/package.json index b5d78de21..76f3136fb 100644 --- a/package.json +++ b/package.json @@ -36,5 +36,8 @@ "stylelint": "^15.5.0", "stylelint-config-standard": "^33.0.0", "stylelint-scss": "^4.6.0" + }, + "dependencies": { + "@react-native-async-storage/async-storage": "^2.1.0" } }