diff options
Diffstat (limited to 'pw_web/log-viewer/src/utils/log-filter')
4 files changed, 849 insertions, 0 deletions
diff --git a/pw_web/log-viewer/src/utils/log-filter/log-filter.models.ts b/pw_web/log-viewer/src/utils/log-filter/log-filter.models.ts new file mode 100644 index 000000000..2fabd7b51 --- /dev/null +++ b/pw_web/log-viewer/src/utils/log-filter/log-filter.models.ts @@ -0,0 +1,61 @@ +// Copyright 2023 The Pigweed Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +export enum ConditionType { + StringSearch, + ColumnSearch, + ExactPhraseSearch, + AndExpression, + OrExpression, + NotExpression, +} + +export type StringSearchCondition = { + type: ConditionType.StringSearch; + searchString: string; +}; + +export type ColumnSearchCondition = { + type: ConditionType.ColumnSearch; + column: string; + value?: string; +}; + +export type ExactPhraseSearchCondition = { + type: ConditionType.ExactPhraseSearch; + exactPhrase: string; +}; + +export type AndExpressionCondition = { + type: ConditionType.AndExpression; + expressions: FilterCondition[]; +}; + +export type OrExpressionCondition = { + type: ConditionType.OrExpression; + expressions: FilterCondition[]; +}; + +export type NotExpressionCondition = { + type: ConditionType.NotExpression; + expression: FilterCondition; +}; + +export type FilterCondition = + | ColumnSearchCondition + | StringSearchCondition + | ExactPhraseSearchCondition + | AndExpressionCondition + | OrExpressionCondition + | NotExpressionCondition; diff --git a/pw_web/log-viewer/src/utils/log-filter/log-filter.ts b/pw_web/log-viewer/src/utils/log-filter/log-filter.ts new file mode 100644 index 000000000..344bba790 --- /dev/null +++ b/pw_web/log-viewer/src/utils/log-filter/log-filter.ts @@ -0,0 +1,254 @@ +// Copyright 2023 The Pigweed Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +import { LogEntry } from '../../shared/interfaces'; +import { FilterCondition, ConditionType } from './log-filter.models'; + +export class LogFilter { + /** + * Generates a structured representation of filter conditions which can be + * used to filter log entries. + * + * @param {string} searchQuery - The search query string provided. + * @returns {function[]} An array of filter functions, each representing a + * set of conditions grouped by logical operators, for filtering log + * entries. + */ + static parseSearchQuery(searchQuery: string): FilterCondition[] { + const filters: FilterCondition[] = []; + const orGroups = searchQuery.split(/\s*\|\s*/); + + for (let i = 0; i < orGroups.length; i++) { + let orGroup = orGroups[i]; + + if (orGroup.includes('(') && !orGroup.includes(')')) { + let j = i + 1; + while (j < orGroups.length && !orGroups[j].includes(')')) { + orGroup += ` | ${orGroups[j]}`; + j++; + } + + if (j < orGroups.length) { + orGroup += ` | ${orGroups[j]}`; + i = j; + } + } + + const andConditions = orGroup.match( + /\([^()]*\)|"[^"]+"|[^\s:]+:[^\s]+|[^\s]+/g, + ); + + const andFilters: FilterCondition[] = []; + + if (andConditions) { + for (const condition of andConditions) { + if (condition.startsWith('(') && condition.endsWith(')')) { + const nestedConditions = condition.slice(1, -1).trim(); + andFilters.push(...this.parseSearchQuery(nestedConditions)); + } else if (condition.startsWith('"') && condition.endsWith('"')) { + const exactPhrase = condition.slice(1, -1).trim(); + andFilters.push({ + type: ConditionType.ExactPhraseSearch, + exactPhrase, + }); + } else if (condition.startsWith('!')) { + const column = condition.slice(1, condition.indexOf(':')); + const value = condition.slice(condition.indexOf(':') + 1); + andFilters.push({ + type: ConditionType.NotExpression, + expression: { + type: ConditionType.ColumnSearch, + column, + value, + }, + }); + } else if (condition.endsWith(':')) { + const column = condition.slice(0, condition.indexOf(':')); + andFilters.push({ + type: ConditionType.ColumnSearch, + column, + }); + } else if (condition.includes(':')) { + const column = condition.slice(0, condition.indexOf(':')); + const value = condition.slice(condition.indexOf(':') + 1); + andFilters.push({ + type: ConditionType.ColumnSearch, + column, + value, + }); + } else { + andFilters.push({ + type: ConditionType.StringSearch, + searchString: condition, + }); + } + } + } + + if (andFilters.length > 0) { + if (andFilters.length === 1) { + filters.push(andFilters[0]); + } else { + filters.push({ + type: ConditionType.AndExpression, + expressions: andFilters, + }); + } + } + } + + if (filters.length === 0) { + filters.push({ + type: ConditionType.StringSearch, + searchString: '', + }); + } + + if (filters.length > 1) { + return [ + { + type: ConditionType.OrExpression, + expressions: filters, + }, + ]; + } + + return filters; + } + + /** + * Takes a condition node, which represents a specific filter condition, and + * recursively generates a filter function that can be applied to log + * entries. + * + * @param {FilterCondition} condition - A filter condition to convert to a + * function. + * @returns {function} A function for filtering log entries based on the + * input condition and its logical operators. + */ + static createFilterFunction( + condition: FilterCondition, + ): (logEntry: LogEntry) => boolean { + switch (condition.type) { + case ConditionType.StringSearch: + return (logEntry) => + this.checkStringInColumns(logEntry, condition.searchString); + case ConditionType.ExactPhraseSearch: + return (logEntry) => + this.checkExactPhraseInColumns(logEntry, condition.exactPhrase); + case ConditionType.ColumnSearch: + return (logEntry) => + this.checkColumn(logEntry, condition.column, condition.value); + case ConditionType.NotExpression: { + const innerFilter = this.createFilterFunction(condition.expression); + return (logEntry) => !innerFilter(logEntry); + } + case ConditionType.AndExpression: { + const andFilters = condition.expressions.map((expr) => + this.createFilterFunction(expr), + ); + return (logEntry) => andFilters.every((filter) => filter(logEntry)); + } + case ConditionType.OrExpression: { + const orFilters = condition.expressions.map((expr) => + this.createFilterFunction(expr), + ); + return (logEntry) => orFilters.some((filter) => filter(logEntry)); + } + default: + // Return a filter that matches all entries + return () => true; + } + } + + /** + * Checks if the column exists in a log entry and then performs a value + * search on the column's value. + * + * @param {LogEntry} logEntry - The log entry to be searched. + * @param {string} column - The name of the column (log entry field) to be + * checked for filtering. + * @param {string} value - An optional string that represents the value used + * for filtering. + * @returns {boolean} True if the specified column exists in the log entry, + * or if a value is provided, returns true if the value matches a + * substring of the column's value (case-insensitive). + */ + private static checkColumn( + logEntry: LogEntry, + column: string, + value?: string, + ): boolean { + const fieldData = logEntry.fields.find((field) => field.key === column); + if (!fieldData) return false; + + if (value === undefined) { + return true; + } + + const searchRegex = new RegExp(value, 'i'); + return searchRegex.test(fieldData.value.toString()); + } + + /** + * Checks if the provided search string exists in any of the log entry + * columns (excluding `severity`). + * + * @param {LogEntry} logEntry - The log entry to be searched. + * @param {string} searchString - The search string to be matched against + * the log entry fields. + * @returns {boolean} True if the search string is found in any of the log + * entry fields, otherwise false. + */ + private static checkStringInColumns( + logEntry: LogEntry, + searchString: string, + ): boolean { + const escapedSearchString = this.escapeRegEx(searchString); + const columnsToSearch = logEntry.fields.filter( + (field) => field.key !== 'severity', + ); + return columnsToSearch.some((field) => + new RegExp(escapedSearchString, 'i').test(field.value.toString()), + ); + } + + /** + * Checks if the exact phrase exists in any of the log entry columns + * (excluding `severity`). + * + * @param {LogEntry} logEntry - The log entry to be searched. + * @param {string} exactPhrase - The exact phrase to search for within the + * log entry columns. + * @returns {boolean} True if the exact phrase is found in any column, + * otherwise false. + */ + private static checkExactPhraseInColumns( + logEntry: LogEntry, + exactPhrase: string, + ): boolean { + const escapedExactPhrase = this.escapeRegEx(exactPhrase); + const searchRegex = new RegExp(escapedExactPhrase, 'i'); + const columnsToSearch = logEntry.fields.filter( + (field) => field.key !== 'severity', + ); + return columnsToSearch.some((field) => + searchRegex.test(field.value.toString()), + ); + } + + private static escapeRegEx(text: string) { + return text.replace(/[-[\]{}()*+?.,\\^$|#\s]/g, '\\$&'); + } +} diff --git a/pw_web/log-viewer/src/utils/log-filter/log-filter_test.ts b/pw_web/log-viewer/src/utils/log-filter/log-filter_test.ts new file mode 100644 index 000000000..63faa39a0 --- /dev/null +++ b/pw_web/log-viewer/src/utils/log-filter/log-filter_test.ts @@ -0,0 +1,173 @@ +// Copyright 2023 The Pigweed Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +import { LogFilter } from './log-filter'; +import { Severity, LogEntry } from '../../shared/interfaces'; +import { describe, expect, test } from '@jest/globals'; +import testData from './test-data'; + +describe('LogFilter', () => { + describe('parseSearchQuery()', () => { + describe('parses search queries correctly', () => { + testData.forEach(({ query, expected }) => { + test(`parses "${query}" correctly`, () => { + const filters = LogFilter.parseSearchQuery(query); + expect(filters).toEqual(expected); + }); + }); + }); + }); + + describe('createFilterFunction()', () => { + describe('filters log entries correctly', () => { + const logEntry1: LogEntry = { + timestamp: new Date(), + severity: Severity.INFO, + fields: [ + { key: 'source', value: 'application' }, + { + key: 'message', + value: 'Request processed successfully!', + }, + ], + }; + + const logEntry2: LogEntry = { + timestamp: new Date(), + severity: Severity.WARNING, + fields: [ + { key: 'source', value: 'database' }, + { + key: 'message', + value: 'Database connection lost. Attempting to reconnect.', + }, + ], + }; + + const logEntry3: LogEntry = { + timestamp: new Date(), + severity: Severity.ERROR, + fields: [ + { key: 'source', value: 'network' }, + { + key: 'message', + value: + 'An unexpected error occurred while performing the operation.', + }, + ], + }; + + const logEntries = [logEntry1, logEntry2, logEntry3]; + + test('should filter by simple string search', () => { + const searchQuery = 'error'; + const filters = LogFilter.parseSearchQuery(searchQuery).map((node) => + LogFilter.createFilterFunction(node), + ); + expect(filters.length).toBe(1); + expect(logEntries.filter(filters[0])).toEqual([logEntry3]); + }); + + test('should filter by column-specific search', () => { + const searchQuery = 'source:database'; + const filters = LogFilter.parseSearchQuery(searchQuery).map((node) => + LogFilter.createFilterFunction(node), + ); + expect(filters.length).toBe(1); + expect(logEntries.filter(filters[0])).toEqual([logEntry2]); + }); + + test('should filter by exact phrase', () => { + const searchQuery = '"Request processed successfully!"'; + const filters = LogFilter.parseSearchQuery(searchQuery).map((node) => + LogFilter.createFilterFunction(node), + ); + expect(filters.length).toBe(1); + expect(logEntries.filter(filters[0])).toEqual([logEntry1]); + }); + + test('should filter by column presence', () => { + const searchQuery = 'source:'; + const filters = LogFilter.parseSearchQuery(searchQuery).map((node) => + LogFilter.createFilterFunction(node), + ); + expect(filters.length).toBe(1); + expect(logEntries.filter(filters[0])).toEqual([ + logEntry1, + logEntry2, + logEntry3, + ]); + }); + + test('should handle AND expressions', () => { + const searchQuery = 'source:network message:error'; + const filters = LogFilter.parseSearchQuery(searchQuery).map((node) => + LogFilter.createFilterFunction(node), + ); + expect(filters.length).toBe(1); + expect(logEntries.filter(filters[0])).toEqual([logEntry3]); + }); + + test('should handle OR expressions', () => { + const searchQuery = 'source:database | source:network'; + const filters = LogFilter.parseSearchQuery(searchQuery).map((node) => + LogFilter.createFilterFunction(node), + ); + expect(filters.length).toBe(1); + expect(logEntries.filter(filters[0])).toEqual([logEntry2, logEntry3]); + }); + + test('should handle NOT expressions', () => { + const searchQuery = '!source:database'; + const filters = LogFilter.parseSearchQuery(searchQuery).map((node) => + LogFilter.createFilterFunction(node), + ); + expect(filters.length).toBe(1); + expect(logEntries.filter(filters[0])).toEqual([logEntry1, logEntry3]); + }); + + test('should handle a combination of AND and OR expressions', () => { + const searchQuery = '(source:database | source:network) message:error'; + const filters = LogFilter.parseSearchQuery(searchQuery).map((node) => + LogFilter.createFilterFunction(node), + ); + expect(filters.length).toBe(1); + expect(logEntries.filter(filters[0])).toEqual([logEntry3]); + }); + + test('should handle a combination of AND, OR, and NOT expressions', () => { + const searchQuery = + '(source:application | source:database) !message:request'; + const filters = LogFilter.parseSearchQuery(searchQuery).map((node) => + LogFilter.createFilterFunction(node), + ); + expect(filters.length).toBe(1); + expect(logEntries.filter(filters[0])).toEqual([logEntry2]); + }); + + test('should handle an empty query', () => { + const searchQuery = ''; + const filters = LogFilter.parseSearchQuery(searchQuery).map((node) => + LogFilter.createFilterFunction(node), + ); + expect(filters.length).toBe(1); + expect(logEntries.filter(filters[0])).toEqual([ + logEntry1, + logEntry2, + logEntry3, + ]); + }); + }); + }); +}); diff --git a/pw_web/log-viewer/src/utils/log-filter/test-data.ts b/pw_web/log-viewer/src/utils/log-filter/test-data.ts new file mode 100644 index 000000000..e7fd7c138 --- /dev/null +++ b/pw_web/log-viewer/src/utils/log-filter/test-data.ts @@ -0,0 +1,361 @@ +// Copyright 2023 The Pigweed Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +import { ConditionType } from './log-filter.models'; + +const testData = [ + { + query: 'error', + expected: [ + { + type: ConditionType.StringSearch, + searchString: 'error', + }, + ], + }, + { + query: 'source:database', + expected: [ + { + type: ConditionType.ColumnSearch, + column: 'source', + value: 'database', + }, + ], + }, + { + query: '"Request processed successfully!"', + expected: [ + { + type: ConditionType.ExactPhraseSearch, + exactPhrase: 'Request processed successfully!', + }, + ], + }, + { + query: 'source:', + expected: [ + { + type: ConditionType.ColumnSearch, + column: 'source', + }, + ], + }, + { + query: 'source:network message:error', + expected: [ + { + type: ConditionType.AndExpression, + expressions: [ + { + type: ConditionType.ColumnSearch, + column: 'source', + value: 'network', + }, + { + type: ConditionType.ColumnSearch, + column: 'message', + value: 'error', + }, + ], + }, + ], + }, + { + query: 'source:database | source:network', + expected: [ + { + type: ConditionType.OrExpression, + expressions: [ + { + type: ConditionType.ColumnSearch, + column: 'source', + value: 'database', + }, + { + type: ConditionType.ColumnSearch, + column: 'source', + value: 'network', + }, + ], + }, + ], + }, + { + query: '!source:database', + expected: [ + { + type: ConditionType.NotExpression, + expression: { + type: ConditionType.ColumnSearch, + column: 'source', + value: 'database', + }, + }, + ], + }, + { + query: 'message:error (source:database | source:network)', + expected: [ + { + type: ConditionType.AndExpression, + expressions: [ + { + type: ConditionType.ColumnSearch, + column: 'message', + value: 'error', + }, + { + type: ConditionType.OrExpression, + expressions: [ + { + type: ConditionType.ColumnSearch, + column: 'source', + value: 'database', + }, + { + type: ConditionType.ColumnSearch, + column: 'source', + value: 'network', + }, + ], + }, + ], + }, + ], + }, + { + query: '(source:database | source:network) message:error', + expected: [ + { + type: ConditionType.AndExpression, + expressions: [ + { + type: ConditionType.OrExpression, + expressions: [ + { + type: ConditionType.ColumnSearch, + column: 'source', + value: 'database', + }, + { + type: ConditionType.ColumnSearch, + column: 'source', + value: 'network', + }, + ], + }, + { + type: ConditionType.ColumnSearch, + column: 'message', + value: 'error', + }, + ], + }, + ], + }, + { + query: '(source:application | source:database) !message:request', + expected: [ + { + type: ConditionType.AndExpression, + expressions: [ + { + type: ConditionType.OrExpression, + expressions: [ + { + type: ConditionType.ColumnSearch, + column: 'source', + value: 'application', + }, + { + type: ConditionType.ColumnSearch, + column: 'source', + value: 'database', + }, + ], + }, + { + type: ConditionType.NotExpression, + expression: { + type: ConditionType.ColumnSearch, + column: 'message', + value: 'request', + }, + }, + ], + }, + ], + }, + { + query: '', + expected: [ + { + type: ConditionType.StringSearch, + searchString: '', + }, + ], + }, + { + // Note: AND takes priority over OR in evaluation. + query: 'source:database message:error | source:network message:error', + expected: [ + { + type: ConditionType.OrExpression, + expressions: [ + { + type: ConditionType.AndExpression, + expressions: [ + { + type: ConditionType.ColumnSearch, + column: 'source', + value: 'database', + }, + { + type: ConditionType.ColumnSearch, + column: 'message', + value: 'error', + }, + ], + }, + { + type: ConditionType.AndExpression, + expressions: [ + { + type: ConditionType.ColumnSearch, + column: 'source', + value: 'network', + }, + { + type: ConditionType.ColumnSearch, + column: 'message', + value: 'error', + }, + ], + }, + ], + }, + ], + }, + { + query: 'source:database | error', + expected: [ + { + type: ConditionType.OrExpression, + expressions: [ + { + type: ConditionType.ColumnSearch, + column: 'source', + value: 'database', + }, + { + type: ConditionType.StringSearch, + searchString: 'error', + }, + ], + }, + ], + }, + { + query: 'source:application request', + expected: [ + { + type: ConditionType.AndExpression, + expressions: [ + { + type: ConditionType.ColumnSearch, + column: 'source', + value: 'application', + }, + { + type: ConditionType.StringSearch, + searchString: 'request', + }, + ], + }, + ], + }, + + { + query: 'source: application request', + expected: [ + { + type: ConditionType.AndExpression, + expressions: [ + { + type: ConditionType.ColumnSearch, + column: 'source', + }, + { + type: ConditionType.StringSearch, + searchString: 'application', + }, + { + type: ConditionType.StringSearch, + searchString: 'request', + }, + ], + }, + ], + }, + { + query: 'source:network | (source:database lorem)', + expected: [ + { + type: ConditionType.OrExpression, + expressions: [ + { + type: ConditionType.ColumnSearch, + column: 'source', + value: 'network', + }, + { + type: ConditionType.AndExpression, + expressions: [ + { + type: ConditionType.ColumnSearch, + column: 'source', + value: 'database', + }, + { + type: ConditionType.StringSearch, + searchString: 'lorem', + }, + ], + }, + ], + }, + ], + }, + { + query: '"unexpected error" "the operation"', + expected: [ + { + type: ConditionType.AndExpression, + expressions: [ + { + type: ConditionType.ExactPhraseSearch, + exactPhrase: 'unexpected error', + }, + { + type: ConditionType.ExactPhraseSearch, + exactPhrase: 'the operation', + }, + ], + }, + ], + }, +]; + +export default testData; |