diff --git a/.vscode/launch.json b/.vscode/launch.json index cfb0844111e..165f42642d4 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -25,7 +25,7 @@ ], "internalConsoleOptions": "openOnSessionStart", "env": { - "COUNTLY_CONFIG_HOSTNAME": "localhost:3001" + "COUNTLY_CONFIG_HOSTNAME": "localhost" } }, { diff --git a/Gruntfile.js b/Gruntfile.js index ab22f5a8943..39dca4dc1d5 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -297,7 +297,13 @@ module.exports = function(grunt) { grunt.registerTask('dist', ['sass', 'concat', 'uglify', 'replace-paths', 'cssmin']); - grunt.registerTask('plugins', 'Minify plugin JS / CSS files and copy images', function() { + grunt.registerTask('compile-plugin-frontends', 'Compile plugin frontend TypeScript to JavaScript', function() { + var execSync = require('child_process').execSync; + execSync('node scripts/compile-plugin-frontends.js', { cwd: __dirname, stdio: 'inherit' }); + }); + + grunt.registerTask('plugins', ['compile-plugin-frontends', 'plugins:run']); + grunt.registerTask('plugins:run', 'Minify plugin JS / CSS files and copy images', function() { var js = [], css = [], img = [], fs = require('fs'), path = require('path'); var pluginFolderPath = path.join(__dirname, 'plugins'); diff --git a/api/ingestor/requestProcessor.ts b/api/ingestor/requestProcessor.ts index eb0fd8b80e0..21db762a32c 100644 --- a/api/ingestor/requestProcessor.ts +++ b/api/ingestor/requestProcessor.ts @@ -234,6 +234,8 @@ interface RequestEvent { _system_event?: boolean; /** ID value */ _id?: string; + /** PII raw delta — original values before obfuscation, keyed by top-level field */ + raw?: Record; } /** @@ -876,6 +878,11 @@ const processToDrill = async function(params: RequestParams, drill_updates: Dril dbEventObject.s = currEvent.sum || 0; dbEventObject.dur = currEvent.dur || 0; dbEventObject.c = currEvent.count || 1; + + if (currEvent.raw && typeof currEvent.raw === 'object') { + dbEventObject.raw = currEvent.raw; + } + eventsToInsert.push({ 'insertOne': { 'document': dbEventObject } }); if (eventKey === '[CLY]_view') { diff --git a/api/ingestor/usage.ts b/api/ingestor/usage.ts index cf099571e4d..a385a597636 100644 --- a/api/ingestor/usage.ts +++ b/api/ingestor/usage.ts @@ -901,7 +901,6 @@ const usage: IngestorUsageModule = { if (Object.keys(update).length > 0) { ob.updates.push(update); } - usage.processCoreMetrics(params); // Collects core metrics }, /** diff --git a/api/utils/eventTransformer.ts b/api/utils/eventTransformer.ts index 8dde76e571d..a817f383d00 100644 --- a/api/utils/eventTransformer.ts +++ b/api/utils/eventTransformer.ts @@ -22,6 +22,8 @@ interface DrillEventDocument { sg?: Record; up_extra?: Record; lu?: number | Date | string; + /** PII raw delta: original values for fields that were obfuscated or blocked. Keys match drill event field names (e.g. sg, up). Only present when PII obfuscation occurred. */ + raw?: Record; [key: string]: unknown; } @@ -44,6 +46,8 @@ interface KafkaEventFormat { sg?: Record; up_extra?: Record; lu?: number; + /** PII raw delta: original values for fields that were obfuscated or blocked. Only present when PII obfuscation occurred. Ignored by the drill_events ClickHouse table (no matching column); consumed by drill_events_raw via a dedicated connector + SMT. */ + raw?: Record; } /** @@ -103,6 +107,9 @@ function transformToKafkaEventFormat(doc: DrillEventDocument | null | undefined) if (doc.up_extra && typeof doc.up_extra === 'object') { result.up_extra = doc.up_extra; } + if (doc.raw && typeof doc.raw === 'object') { + result.raw = doc.raw; + } // Optional date field if (doc.lu !== undefined && doc.lu !== null) { diff --git a/bin/datasource/Dockerfile-kafka-connect-clickhouse-dev b/bin/datasource/Dockerfile-kafka-connect-clickhouse-dev index 9bc3e75ce48..59a5837a6a5 100644 --- a/bin/datasource/Dockerfile-kafka-connect-clickhouse-dev +++ b/bin/datasource/Dockerfile-kafka-connect-clickhouse-dev @@ -1,31 +1,23 @@ -FROM eclipse-temurin:21-jre +# Stage 1: compile the RawEventExtractor SMT against the Kafka Connect API JARs +# that are bundled inside the existing runtime image. +FROM eclipse-temurin:21-jdk AS smt-builder -ARG SCALA_VERSION=2.13 -ARG KAFKA_VERSION=4.0.0 -ARG CH_SINK_VERSION=1.3.5 +# Copy the Kafka libs from the runtime image so javac has the Connect API on its classpath +COPY --from=countly/kafka-connect-clickhouse:latest /opt/kafka/libs /opt/kafka/libs +COPY --from=countly/kafka-connect-clickhouse:latest /opt/kafka/plugins /opt/kafka/plugins -# Install Kafka (includes connect-distributed.sh) -RUN mkdir -p /opt && \ - curl -fSL https://dlcdn.apache.org/kafka/${KAFKA_VERSION}/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz \ - -o /tmp/kafka.tgz && \ - tar -xzf /tmp/kafka.tgz -C /opt && \ - ln -s /opt/kafka_${SCALA_VERSION}-${KAFKA_VERSION} /opt/kafka && \ - rm /tmp/kafka.tgz +COPY transforms/RawEventExtractor.java /tmp/RawEventExtractor.java +RUN mkdir -p /tmp/smt-build && \ + javac -cp "/opt/kafka/libs/*:/opt/kafka/plugins/clickhouse/*" \ + -d /tmp/smt-build \ + /tmp/RawEventExtractor.java && \ + jar cf /tmp/countly-smt.jar -C /tmp/smt-build . -# Plugins -RUN mkdir -p /opt/kafka/plugins/clickhouse -# Pull the ClickHouse Sink (Apache-2.0) release ZIP and extract jars -RUN apt-get update && apt-get install -y unzip ca-certificates curl && \ - curl -fSL \ - https://github.com/ClickHouse/clickhouse-kafka-connect/releases/download/v${CH_SINK_VERSION}/clickhouse-kafka-connect-v${CH_SINK_VERSION}.zip \ - -o /tmp/ch-sink.zip && \ - unzip -q /tmp/ch-sink.zip -d /opt/kafka/plugins/clickhouse && \ - rm -rf /var/lib/apt/lists/* /tmp/ch-sink.zip +# Stage 2: extend the existing runtime image with the compiled JAR +FROM countly/kafka-connect-clickhouse:latest + +COPY --from=smt-builder /tmp/countly-smt.jar /opt/kafka/plugins/countly-smt/countly-smt.jar # Copy minimal config (environment variables will override) COPY connect-distributed.properties /opt/kafka/config/connect-distributed.properties COPY clickhouse-connector.json /opt/kafka/config/clickhouse-connector.json - -EXPOSE 8083 -WORKDIR /opt/kafka -CMD ["bin/connect-distributed.sh", "config/connect-distributed.properties"] \ No newline at end of file diff --git a/bin/datasource/clickhouse-connector-raw.json b/bin/datasource/clickhouse-connector-raw.json new file mode 100644 index 00000000000..1f0416dd0be --- /dev/null +++ b/bin/datasource/clickhouse-connector-raw.json @@ -0,0 +1,26 @@ +{ + "name": "clickhouse-sink-raw", + "config": { + "connector.class": "com.clickhouse.kafka.connect.ClickHouseSinkConnector", + "tasks.max": "1", + "topics": "drill-events", + "hostname": "clickhouse", + "port": "8123", + "database": "countly_drill", + "username": "default", + "password": "", + "ssl": "false", + "exactlyOnce": "false", + "topic2TableMap": "drill-events=drill_events_raw", + "clickhouseSettings": "input_format_binary_read_json_as_string=1,allow_experimental_json_type=1,enable_json_type=1,async_insert=1,wait_for_async_insert=1", + "bypassRowBinary": "false", + "errors.tolerance": "none", + "errors.retry.timeout": "60", + "tableRefreshInterval": "300", + "key.converter": "org.apache.kafka.connect.storage.StringConverter", + "value.converter": "org.apache.kafka.connect.json.JsonConverter", + "value.converter.schemas.enable": "false", + "transforms": "extractRaw", + "transforms.extractRaw.type": "dev.you.smt.RawEventExtractor" + } +} diff --git a/bin/datasource/docker-compose.yml b/bin/datasource/docker-compose.yml index ec7ec9f23c7..a8185bd92b0 100644 --- a/bin/datasource/docker-compose.yml +++ b/bin/datasource/docker-compose.yml @@ -36,7 +36,9 @@ services: # Kafka Connect with ClickHouse connector kafka-connect: - image: countly/kafka-connect-clickhouse:latest + build: + context: . + dockerfile: Dockerfile-kafka-connect-clickhouse-dev container_name: kafka-connect depends_on: kafka: @@ -80,6 +82,9 @@ services: CONNECT_CONSUMER_MAX_PARTITION_FETCH_BYTES: "52428800" # 50MB per partition CONNECT_PRODUCER_MAX_REQUEST_SIZE: "52428800" # 50MB producer request CONNECT_PRODUCER_BUFFER_MEMORY: "67108864" # 64MB producer buffer + # Small heap to force frequent GC — for benchmarking SMT GC overhead only + #KAFKA_HEAP_OPTS: "-Xms128M -Xmx512M" + #KAFKA_JVM_PERFORMANCE_OPTS: "-server -XX:+UseG1GC -XX:MaxGCPauseMillis=20 -XX:InitiatingHeapOccupancyPercent=35 -Xlog:gc*:file=/tmp/gc.log:time,level,tags:filecount=1,filesize=50m" networks: - kafka-network @@ -130,7 +135,7 @@ services: sleep 5 done echo 'Connect is ready, creating connector...' - + curl -X PUT -H 'Content-Type: application/json' \ -d '{ "connector.class": "com.clickhouse.kafka.connect.ClickHouseSinkConnector", @@ -154,13 +159,45 @@ services: "errors.deadletterqueue.topic.replication.factor": "1" }' \ http://kafka-connect:8083/connectors/clickhouse-sink/config - - if [ $? -eq 0 ]; then - echo 'Connector created successfully with DLQ!' - else - echo 'Failed to create connector' + + if [ $? -ne 0 ]; then + echo 'Failed to create clickhouse-sink connector' exit 1 fi + echo 'clickhouse-sink connector created successfully!' + + echo 'Creating clickhouse-sink-raw connector...' + curl -X PUT -H 'Content-Type: application/json' \ + -d '{ + "connector.class": "com.clickhouse.kafka.connect.ClickHouseSinkConnector", + "tasks.max": "1", + "topics": "drill-events", + "hostname": "clickhouse", + "port": "8123", + "ssl": "false", + "database": "countly_drill", + "username": "default", + "password": "", + "topic2TableMap": "drill-events=drill_events_raw", + "key.converter": "org.apache.kafka.connect.storage.StringConverter", + "value.converter": "org.apache.kafka.connect.json.JsonConverter", + "value.converter.schemas.enable": "false", + "errors.tolerance": "all", + "errors.retry.timeout": "300000", + "errors.retry.delay.max.ms": "10000", + "errors.deadletterqueue.topic.name": "drill-events-dlq", + "errors.deadletterqueue.context.headers.enable": "true", + "errors.deadletterqueue.topic.replication.factor": "1", + "transforms": "extractRaw", + "transforms.extractRaw.type": "dev.you.smt.RawEventExtractor" + }' \ + http://kafka-connect:8083/connectors/clickhouse-sink-raw/config + + if [ $? -ne 0 ]; then + echo 'Failed to create clickhouse-sink-raw connector' + exit 1 + fi + echo 'clickhouse-sink-raw connector created successfully!' restart: "no" # Kafka UI for monitoring @@ -216,7 +253,7 @@ services: sleep 2 done echo 'MongoDB is ready, initializing replica set...' - + mongosh --host mongodb:27017 --eval ' try { rs.status(); @@ -232,7 +269,7 @@ services: console.log("Replica set initialized successfully"); } ' - + echo 'Waiting for replica set to be ready...' mongosh --host mongodb:27017 --eval ' while (true) { diff --git a/bin/datasource/transforms/RawEventExtractor.java b/bin/datasource/transforms/RawEventExtractor.java new file mode 100644 index 00000000000..72e069af121 --- /dev/null +++ b/bin/datasource/transforms/RawEventExtractor.java @@ -0,0 +1,181 @@ +package dev.you.smt; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.kafka.common.config.ConfigDef; +import org.apache.kafka.connect.connector.ConnectRecord; +import org.apache.kafka.connect.transforms.Transformation; + +import java.util.Map; + +/** + * Kafka Connect Single Message Transform (SMT) for the countly-sink-raw connector. + * + * Purpose: consumes drill-events Kafka messages that contain a `raw` delta field + * (added by the PII processor when obfuscation occurred) and reconstructs the full + * original event so it can be written to the drill_events_raw ClickHouse table. + * + * Behaviour: + * - If the message has no `raw` field → returns null (tombstone), skipping the record. + * Only events that were PII-obfuscated get written to drill_events_raw. + * - If the message has a `raw` field → deep-merges each sub-object in `raw` back into + * the corresponding top-level field of the event, then removes the `raw` field. + * Example: raw.sg.vin = "123" is merged into sg.vin, restoring the original value. + * + * Deployment: place the compiled JAR in the Kafka Connect plugin path alongside the + * ClickHouse sink connector JAR (see Dockerfile-kafka-connect-clickhouse-dev). + * Register via clickhouse-connector-raw.json: + * "transforms": "extractRaw", + * "transforms.extractRaw.type": "dev.you.smt.RawEventExtractor" + */ +public class RawEventExtractor> implements Transformation { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + /** + * Recursively merge all entries from source into target. + * For keys present in both where both values are objects, recurse. + * For all other cases, set the target key to the source value. + */ + private static void deepMergeSg(ObjectNode target, JsonNode source) { + source.fields().forEachRemaining(entry -> { + String key = entry.getKey(); + JsonNode srcVal = entry.getValue(); + JsonNode tgtVal = target.get(key); + if (srcVal.isObject() && tgtVal != null && tgtVal.isObject()) { + deepMergeSg((ObjectNode) tgtVal, srcVal); + } else { + target.set(key, srcVal); + } + }); + } + + @Override + public R apply(R record) { + if (record.value() == null) { + return null; + } + + try { + // Parse value — KafkaEventSink serialises events as plain JSON strings. + String json = record.value() instanceof String + ? (String) record.value() + : MAPPER.writeValueAsString(record.value()); + + JsonNode root = MAPPER.readTree(json); + + if (!root.isObject()) { + return null; + } + + ObjectNode event = (ObjectNode) root; + + // Filter: skip events that were never PII-obfuscated. + if (!event.has("raw") || event.get("raw").isNull()) { + return null; + } + + JsonNode rawDelta = event.get("raw"); + + if (!rawDelta.isObject()) { + return null; + } + + // Step 1: rename obfuscated key aliases back to their original names in place. + // Done before the value merge so that: + // - key-only renames: value is preserved as-is (no raw.sg entry for these) + // - key+value renames: value under the original key is then overwritten by deepMergeSg + // Each entry: { "path": ["parent", ...], "original": "vin", "obfuscated": "v**" } + if (rawDelta.has("sg_renames") && rawDelta.get("sg_renames").isArray()) { + JsonNode existingSg = event.get("sg"); + if (existingSg != null && existingSg.isObject()) { + rawDelta.get("sg_renames").forEach(rename -> { + JsonNode parent = existingSg; + JsonNode pathArr = rename.get("path"); + if (pathArr != null && pathArr.isArray()) { + for (JsonNode step : pathArr) { + if (parent == null || !parent.isObject()) { + parent = null; + break; + } + parent = parent.get(step.asText()); + } + } + if (parent != null && parent.isObject()) { + JsonNode obfuscatedNode = rename.get("obfuscated"); + JsonNode originalNode = rename.get("original"); + if (obfuscatedNode != null && originalNode != null) { + String obfuscated = obfuscatedNode.asText(); + String original = originalNode.asText(); + JsonNode value = parent.get(obfuscated); + if (value != null) { + ((ObjectNode) parent).set(original, value); + ((ObjectNode) parent).remove(obfuscated); + } + } + } + }); + } + } + + // Step 2: deep-merge sparse value delta (only keys whose values were obfuscated/blocked). + // For key+value obfuscation this overwrites the still-obfuscated value moved in step 1. + if (rawDelta.has("sg") && rawDelta.get("sg").isObject()) { + JsonNode sgDelta = rawDelta.get("sg"); + JsonNode existingSg = event.get("sg"); + if (existingSg != null && existingSg.isObject()) { + deepMergeSg((ObjectNode) existingSg, sgDelta); + } else { + event.set("sg", sgDelta); + } + } + + // Restore all other top-level fields (e.g. "n" for the event key). + rawDelta.fields().forEachRemaining(entry -> { + String fieldName = entry.getKey(); + if (fieldName.equals("sg") || fieldName.equals("sg_renames")) { + return; + } + JsonNode rawValue = entry.getValue(); + if (rawValue == null || rawValue.isNull()) { + return; + } + event.set(fieldName, rawValue); + }); + + // Remove the raw delta field so ClickHouse doesn't see it. + event.remove("raw"); + + return record.newRecord( + record.topic(), + record.kafkaPartition(), + record.keySchema(), + record.key(), + record.valueSchema(), + event.toString(), + record.timestamp() + ); + + } + catch (Exception e) { + // On any parse failure, skip the record rather than failing the connector. + return null; + } + } + + @Override + public ConfigDef config() { + return new ConfigDef(); + } + + @Override + public void configure(Map configs) { + // No configuration required. + } + + @Override + public void close() { + // Nothing to close. + } +} diff --git a/frontend/express/public/javascripts/countly/vue/components/datatable.js b/frontend/express/public/javascripts/countly/vue/components/datatable.js index dfb067d1e16..fc1db645d98 100644 --- a/frontend/express/public/javascripts/countly/vue/components/datatable.js +++ b/frontend/express/public/javascripts/countly/vue/components/datatable.js @@ -315,11 +315,14 @@ if (newDataView) { const hasCursorData = newDataView.hasNextPage !== undefined || newDataView.nextCursor !== undefined; const shouldUseCursor = hasCursorData && (newDataView.hasNextPage || newDataView.nextCursor); + const hasActiveCursorState = !!this.controlParams.cursor || + (this.controlParams.cursorHistory && this.controlParams.cursorHistory.length > 0) || + this.controlParams.page > this.firstPage; if (shouldUseCursor && !this.controlParams.useCursorPagination) { this.controlParams.useCursorPagination = true; } - else if (!shouldUseCursor && this.controlParams.useCursorPagination) { + else if (!shouldUseCursor && this.controlParams.useCursorPagination && !hasActiveCursorState) { this.controlParams.useCursorPagination = false; } } @@ -400,11 +403,16 @@ methods: { checkPageBoundaries() { if (this.lastPage > 0 && this.controlParams.page > this.lastPage) { - this.controlParams.page = this.lastPage; + this.updateControlParams({ + page: this.lastPage + }); + return; } if (this.controlParams.page < 1) { - this.controlParams.page = 1; + this.updateControlParams({ + page: 1 + }); } }, @@ -446,6 +454,7 @@ try { if ( countlyGlobal.member.columnOrder && + countlyGlobal.member.columnOrder[this.persistKey] && countlyGlobal.member.columnOrder[this.persistKey].tableSortMap ) { defaultState.selectedDynamicCols = countlyGlobal.member.columnOrder[this.persistKey].tableSortMap; @@ -534,8 +543,10 @@ }); } else { - // Fallback: just go back one page (will show first page data) - this.controlParams.page--; + // Fallback: go back one page through unified updater so page-1 cursor state is normalized. + this.updateControlParams({ + page: this.controlParams.page - 1 + }); } } else { @@ -654,6 +665,11 @@ }, updateControlParams(newParams) { + // Keep cursor state consistent: page 1 must never carry a cursor/history token. + if (newParams && Number(newParams.page) === this.firstPage) { + newParams.cursor = null; + newParams.cursorHistory = []; + } _.extend(this.controlParams, newParams); }, diff --git a/package.json b/package.json index f3fe67b688f..23495896f82 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,9 @@ "start:jobserver:dev": "COUNTLY_CONFIG__SYMLINKED=true COUNTLY_CONTAINER=api node --preserve-symlinks --preserve-symlinks-main --trace-warnings jobServer/index.js", "start:all:dev": "npm run start:api:dev & npm run start:frontend:dev & npm run start:ingestor:dev & npm run start:aggregator:dev & npm run start:jobserver:dev & wait", "test:services": "env-cmd -f test/configs/base.env -f test/configs/suite-overrides/core.env sh -c 'npm run start:api:dev & npm run start:frontend:dev & npm run start:ingestor:dev & npm run start:aggregator:dev & npm run start:jobserver:dev & wait'", - "postinstall": "mkdir -p frontend/express/public/sdk/web && cp -rf node_modules/countly-sdk-web/lib/* frontend/express/public/sdk/web/" + "postinstall": "mkdir -p frontend/express/public/sdk/web && cp -rf node_modules/countly-sdk-web/lib/* frontend/express/public/sdk/web/", + "build:plugin-frontends": "node scripts/compile-plugin-frontends.js", + "watch:plugin-frontend": "node scripts/compile-plugin-frontends.js --watch" }, "lint-staged": { "*.{js,vue}": [ diff --git a/plugins/alerts/api/alertModules/pii.js b/plugins/alerts/api/alertModules/pii.js new file mode 100644 index 00000000000..9d5b72ef68a --- /dev/null +++ b/plugins/alerts/api/alertModules/pii.js @@ -0,0 +1,153 @@ +/** + * @typedef {import('../parts/common-lib.js').App} App + */ + +const log = require('../../../../api/utils/log.js')('alert:pii'); +const moment = require('moment-timezone'); +const common = require('../../../../api/utils/common.js'); +const commonLib = require("../parts/common-lib.js"); +const { ObjectId } = require('mongodb'); + +module.exports.triggerByEvent = triggerByEvent; +/** + * Checks if given payload contains PII incidents and triggers matching alerts. + * @param {object} payload - { incidents, app } + */ +async function triggerByEvent(payload) { + const incidents = payload?.incidents; + const app = payload?.app; + + if (!incidents || !Array.isArray(incidents) || !incidents.length || !app) { + return; + } + + // Find alerts for this specific app AND alerts targeting all apps + const [appAlerts, allAppsAlerts] = await Promise.all([ + common.readBatcher.getMany("alerts", { + selectedApps: app._id.toString(), + alertDataType: "pii", + alertDataSubType: commonLib.TRIGGERED_BY_EVENT.pii, + enabled: true, + }), + common.readBatcher.getMany("alerts", { + selectedApps: "all", + alertDataType: "pii", + alertDataSubType: commonLib.TRIGGERED_BY_EVENT.pii, + enabled: true, + }), + ]); + + const alerts = [...(appAlerts || []), ...(allAppsAlerts || [])]; + if (!alerts.length) { + return; + } + + await Promise.all(alerts.map(alert => { + // If alert targets a specific rule, only trigger for incidents matching that rule + if (alert.alertDataSubType2) { + const matchingIncidents = incidents.filter(inc => inc.ruleId === alert.alertDataSubType2); + if (!matchingIncidents.length) { + return Promise.resolve(); + } + return commonLib.trigger({ + alert, + app, + date: new Date(), + extra: { + incidentCount: matchingIncidents.length, + firstIncident: matchingIncidents[0], + } + }, log); + } + return commonLib.trigger({ + alert, + app, + date: new Date(), + extra: { + incidentCount: incidents.length, + firstIncident: incidents[0], + } + }, log); + })); +} + + +module.exports.check = async function({ alertConfigs: alert, scheduledTo: date }) { + const selectedApp = alert.selectedApps[0]; + let app = null; + let appId = null; + + if (selectedApp !== "all") { + app = await common.readBatcher.getOne("apps", { _id: new ObjectId(selectedApp) }); + if (!app) { + log.e(`App ${selectedApp} couldn't be found`); + return; + } + appId = app._id.toString(); + } + + let { period, compareType, compareValue, filterValue, alertDataSubType2 } = alert; + compareValue = Number(compareValue); + + const metricValue = await countIncidents(date, period, appId, filterValue, alertDataSubType2) || 0; + + if (compareType === commonLib.COMPARE_TYPE_ENUM.MORE_THAN) { + if (metricValue > compareValue) { + await commonLib.trigger({ alert, app, metricValue, date }, log); + } + } + else { + const before = moment(date).subtract(1, commonLib.PERIOD_TO_DATE_COMPONENT_MAP[period]).toDate(); + const metricValueBefore = await countIncidents(before, period, appId, filterValue, alertDataSubType2); + if (!metricValueBefore) { + return; + } + + const change = (metricValue / metricValueBefore - 1) * 100; + const shouldTrigger = compareType === commonLib.COMPARE_TYPE_ENUM.INCREASED_BY + ? change >= compareValue + : change <= -compareValue; + + if (shouldTrigger) { + await commonLib.trigger({ alert, app, date, metricValue, metricValueBefore }, log); + } + } +}; + +/** + * Count PII incidents within a time period. + * @param {Date} date - end date of the period + * @param {string} period - hourly|daily|monthly + * @param {string|null} appId - app ID to filter by, or null for all apps + * @param {string|undefined} actionFilter - optional action filter (NOTIFY|OBFUSCATE|BLOCK) + * @param {string|undefined} ruleId - optional PII rule ID to filter by + * @returns {Promise} - incident count + */ +async function countIncidents(date, period, appId, actionFilter, ruleId) { + const periodMs = { + hourly: 60 * 60 * 1000, + daily: 24 * 60 * 60 * 1000, + monthly: 30 * 24 * 60 * 60 * 1000, + }; + + const endTs = date.getTime(); + const startTs = endTs - (periodMs[period] || periodMs.daily); + + const query = { + ts: { $gte: startTs, $lte: endTs }, + }; + + if (appId) { + query.app_id = appId; + } + + if (actionFilter) { + query.action = actionFilter; + } + + if (ruleId) { + query.ruleId = ruleId; + } + + return common.db.collection("pii_incidents").countDocuments(query); +} diff --git a/plugins/alerts/api/ingestor.js b/plugins/alerts/api/ingestor.js index f792be4d077..79b7bce8e9a 100644 --- a/plugins/alerts/api/ingestor.js +++ b/plugins/alerts/api/ingestor.js @@ -20,7 +20,7 @@ const commonLib = require("./parts/common-lib.js"); } for (let { module, name } of TRIGGER_BY_EVENT) { - if (name !== "crashes") { + if (name !== "crashes" && name !== "pii") { try { await module.triggerByEvent({ events, app }); } @@ -31,6 +31,19 @@ const commonLib = require("./parts/common-lib.js"); } }); + plugins.register("/pii/incident", async function(ob) { + for (let { module, name } of TRIGGER_BY_EVENT) { + if (name === "pii") { + try { + await module.triggerByEvent(ob.data); + } + catch (err) { + log.e("Alert module 'pii' couldn't be triggered by event", err); + } + } + } + }); + }(exported)); module.exports = exported; \ No newline at end of file diff --git a/plugins/alerts/api/jobs/AlertProcessor.js b/plugins/alerts/api/jobs/AlertProcessor.js index 507904935e1..5e0e06826a8 100644 --- a/plugins/alerts/api/jobs/AlertProcessor.js +++ b/plugins/alerts/api/jobs/AlertProcessor.js @@ -17,6 +17,7 @@ const ALERT_MODULES = { "cohorts": require("../alertModules/cohorts.js"), "dataPoints": require("../alertModules/dataPoints.js"), "crashes": require("../alertModules/crashes.js"), + "pii": require("../alertModules/pii.js"), }; /** diff --git a/plugins/alerts/api/parts/common-lib.js b/plugins/alerts/api/parts/common-lib.js index 3c21b52f1ea..c3def8b494f 100644 --- a/plugins/alerts/api/parts/common-lib.js +++ b/plugins/alerts/api/parts/common-lib.js @@ -18,7 +18,7 @@ * @property {boolean} enabled - true|false * @property {string} compareDescribe - text to show on lists for this alert * @property {Array} alertValues - audience e.g. for alertBy="email", list of e-mails - * @property {Array} allGroups - + * @property {Array} allGroups - * @property {string} createdBy - creation time */ @@ -81,6 +81,7 @@ const TRIGGERED_BY_EVENT = { nps: "new NPS response", rating: "new rating response", crashes: "new crash/error", + pii: "new sensitive data incident", }; module.exports = { @@ -207,7 +208,7 @@ async function compileEmail(result) { * Formats the metric value to ensure it maintains its type. * If the value is a number, it rounds to 2 decimal places if necessary. * Otherwise, it returns the value as is. - * + * * @param {number|string} value - The value to be formatted. * @returns {number|string} The formatted value, maintaining the original type. */ diff --git a/plugins/alerts/frontend/public/javascripts/countly.models.js b/plugins/alerts/frontend/public/javascripts/countly.models.js index 0db22515e39..1cc2384d400 100644 --- a/plugins/alerts/frontend/public/javascripts/countly.models.js +++ b/plugins/alerts/frontend/public/javascripts/countly.models.js @@ -158,6 +158,38 @@ } countlyAlerts.getRatingForApp = getRatingForApp; + /** + * Get PII rules for the specified app. + * @param {string} appId - The ID of the app. + * @param {function} callback - The callback function. + */ + function getPiiRulesForApp(appId, callback) { + var data = { + enabled: "true", + }; + if (appId === "all") { + data.app_ids = "all"; + data.scope_filter = "global"; + } + else { + data.app_ids = JSON.stringify([appId]); + data.include_global = "true"; + } + $.ajax({ + type: "GET", + url: countlyCommon.API_PARTS.data.r + "/pii/rules", + data: data, + dataType: "json", + success: function(res) { + if (res && Array.isArray(res)) { + return callback(res); + } + return callback([]); + }, + }); + } + countlyAlerts.getPiiRulesForApp = getPiiRulesForApp; + /** * extract getEventLongName * @param {string} eventKey - event key in db diff --git a/plugins/alerts/frontend/public/javascripts/countly.views.js b/plugins/alerts/frontend/public/javascripts/countly.views.js index ec1c73a7cf1..037356370bd 100644 --- a/plugins/alerts/frontend/public/javascripts/countly.views.js +++ b/plugins/alerts/frontend/public/javascripts/countly.views.js @@ -49,6 +49,7 @@ saveButtonLabel: "", apps: [""], allowAll: false, + isAllAppsSelected: false, filterButton: false, showSubType1: true, showSubType2: false, @@ -196,6 +197,18 @@ }, ], }, + pii: { + target: [ + { + value: "# of sensitive data incidents", + label: jQuery.i18n.map["alert.pii-incidents-count"] || "# of incidents", + }, + { + value: "new sensitive data incident", + label: jQuery.i18n.map["alert.new-pii-incident"] || "new incident", + }, + ], + }, revenue: { target: [ { value: "total revenue", label: jQuery.i18n.map["alert.total-revenue"] || "total revenue" }, @@ -275,6 +288,7 @@ "new NPS response", "new rating response", "new crash/error", + "new sensitive data incident", "o", "m", ]; @@ -292,6 +306,7 @@ "new NPS response", "new rating response", "new crash/error", + "new sensitive data incident", "o", "m", ]; @@ -357,6 +372,7 @@ { label: jQuery.i18n.map["alert.Survey"], value: "survey" }, { label: jQuery.i18n.map["alert.User"], value: "users" }, { label: jQuery.i18n.map["alert.View"], value: "views" }, + { label: jQuery.i18n.map["alert.PII"] || "Sensitive Data", value: "pii" }, ]; // disable enterprise plugins if they are not available if (!countlyGlobal.plugins.includes("concurrent_users")) { @@ -374,6 +390,13 @@ if (!countlyGlobal.plugins.includes("users")) { alertDataTypeOptions = alertDataTypeOptions.filter(({ value }) => value !== "users"); } + if (!countlyGlobal.plugins.includes("pii")) { + alertDataTypeOptions = alertDataTypeOptions.filter(({ value }) => value !== "pii"); + } + // When "All Applications" is selected, only dataPoints and pii are available + if (this.isAllAppsSelected) { + alertDataTypeOptions = alertDataTypeOptions.filter(({ value }) => value === "dataPoints" || value === "pii"); + } return alertDataTypeOptions; }, alertDefine: function() { @@ -488,6 +511,8 @@ return "Widget Name"; case "rating": return "Widget Name"; + case "pii": + return "Rule"; } }, showFilterButton: function(obj) { @@ -505,7 +530,7 @@ getMetrics: function() { const formData = this.$refs.drawerData.editedObject; this.alertDataSubType2Options = []; - if (formData.selectedApps === 'all') { + if (formData.selectedApps === 'all' && formData.alertDataType !== 'pii') { formData.alertDataType = 'dataPoints'; formData.alertDataSubType = 'total data points'; } @@ -604,8 +629,20 @@ } ); } + if (formData.alertDataType === "pii") { + var piiAppId = formData.selectedApps === 'all' ? 'all' : formData.selectedApps; + countlyAlerts.getPiiRulesForApp( + piiAppId, + (data) => { + this.alertDataSubType2Options = data.map((r) => { + return { value: r._id, label: countlyCommon.unescapeHtml(r.name) }; + }); + } + ); + } }, appSelected: function() { + this.isAllAppsSelected = this.$refs.drawerData.editedObject.selectedApps === 'all'; this.resetAlertCondition(); this.getMetrics(); }, @@ -626,6 +663,7 @@ "survey", "nps", "rating", + "pii", ]; if (validDataTypesForSubType2.includes(val)) { this.showSubType2 = true; @@ -746,6 +784,8 @@ return "cly-io-16 cly-is cly-is-users"; case "views": return "cly-io-16 cly-is cly-is-eye"; + case "pii": + return "cly-io-16 cly-is cly-is-shield-exclamation"; } }, handleFilterClosing: function() { @@ -1042,6 +1082,7 @@ this.showCondition = false; this.showConditionValue = false; newState.selectedApps = newState.selectedApps[0]; + this.isAllAppsSelected = newState.selectedApps === 'all'; newState.alertName = countlyCommon.unescapeHtml(newState.alertName); newState.alertValues = newState.alertValues.map((email) => countlyCommon.unescapeHtml(email)); this.getMetrics(); @@ -1087,12 +1128,21 @@ return true; } }, - calculateWidth(value) { + calculateWidth(value, options) { if (!value) { return; } + var displayText = value; + if (options && Array.isArray(options)) { + var match = options.find(function(o) { + return o.value === value; + }); + if (match && match.label) { + displayText = match.label; + } + } let tmpEl = document.createElement("span"); - tmpEl.textContent = value; + tmpEl.textContent = displayText; tmpEl.style.cssText = ` visibility: hidden; position: fixed; @@ -1347,6 +1397,45 @@ beforeCreate: function() { this.$store.dispatch("countlyAlerts/initialize"); }, + mounted: function() { + var self = this; + var prefill = sessionStorage.getItem('alerts_prefill'); + if (prefill) { + sessionStorage.removeItem('alerts_prefill'); + var data = JSON.parse(prefill); + if (data.alertId) { + var unwatch = this.$watch( + function() { + return self.$store.getters["countlyAlerts/table/getInitialized"]; + }, + function(initialized) { + if (initialized) { + unwatch(); + var rows = self.$store.getters["countlyAlerts/table/all"]; + var alertRow = rows.find(function(r) { + return r._id === data.alertId; + }); + if (alertRow) { + self.openDrawer("home", Object.assign({}, alertRow)); + } + } + }, + { immediate: true } + ); + } + else if (data.alertDataType) { + var config = countlyAlerts.defaultDrawerConfigValue(); + config.alertDataType = data.alertDataType; + if (data.alertDataSubType2) { + config.alertDataSubType2 = data.alertDataSubType2; + } + if (data.selectedApps) { + config.selectedApps = [data.selectedApps]; + } + this.openDrawer("home", config); + } + } + }, methods: { createAlert: function() { var config = countlyAlerts.defaultDrawerConfigValue(); diff --git a/plugins/alerts/frontend/public/localization/alerts.properties b/plugins/alerts/frontend/public/localization/alerts.properties index 8a576060271..036e02f0e25 100644 --- a/plugins/alerts/frontend/public/localization/alerts.properties +++ b/plugins/alerts/frontend/public/localization/alerts.properties @@ -81,7 +81,7 @@ alert.add-alert=Add Alert alert.save-alert=Save Alert alert.save=Create alert.tips = Overview of all alerts set up. Create new alerts to receive emails when specific conditions related to metrics are met. -alerts.application-tooltip= Data Points is the only data type available for your selected Application. +alerts.application-tooltip= Only Data Points and Sensitive Data are available when All Applications is selected. alerts.update-status-success = Successfully update status alerts.empty-title= ...hmm, seems empty here alerts.empty-action-button-title=+ Create New Alert @@ -143,6 +143,9 @@ alert.total-revenue = total revenue alert.average-revenue-per-user = average revenue per user alert.average-revenue-per-paying-user = average revenue per paying user alert.paying-users-count = # of paying users +alert.PII = Sensitive Data +alert.pii-incidents-count = # of incidents +alert.new-pii-incident = new incident alerts.create-alert-success = Alert has been created successfully alerts.create-alert-fail = The alert could not be created. Please try again or check the server logs diff --git a/plugins/alerts/frontend/public/stylesheets/vue-main.scss b/plugins/alerts/frontend/public/stylesheets/vue-main.scss index 082186f3957..ccc231a3e97 100644 --- a/plugins/alerts/frontend/public/stylesheets/vue-main.scss +++ b/plugins/alerts/frontend/public/stylesheets/vue-main.scss @@ -342,3 +342,15 @@ background-position: center; display: inline-block; } + +.cly-is-shield-exclamation:before { + color: #FF9382; + background-color: #FCF1F0; + width: 16px; + line-height: 16px; + height: 16px; + background-size: 16px 16px; + border-radius: 4px; + background-position: center; + display: inline-block; +} diff --git a/plugins/alerts/frontend/public/templates/vue-main.html b/plugins/alerts/frontend/public/templates/vue-main.html index d0e02499ac8..c73e4b64b40 100644 --- a/plugins/alerts/frontend/public/templates/vue-main.html +++ b/plugins/alerts/frontend/public/templates/vue-main.html @@ -52,7 +52,7 @@ {{i18n('alerts.status-all')}} {{i18n('alerts.status-enabled')}} - {{i18n('alerts.status-disabled')}} + {{i18n('alerts.status-disabled')}}
@@ -175,7 +175,6 @@