diff --git a/backend/internal/openappsec-log.js b/backend/internal/openappsec-log.js index a695826a..22def5c3 100755 --- a/backend/internal/openappsec-log.js +++ b/backend/internal/openappsec-log.js @@ -14,7 +14,7 @@ const internalOpenappsecLog = { * @param {String} [search_query] * @returns {Promise} */ - getAll: (access, expand, search_query) => { + getAllold: (access, expand, search_query) => { return access.can('auditlog:list') .then(() => { @@ -37,6 +37,7 @@ const internalOpenappsecLog = { return path.extname(a).localeCompare(path.extname(b)); }); + // Group the log files by their base name const groupedFiles = sortedLogFiles.reduce((groups, file) => { const fileName = path.basename(file, path.extname(file)); if (!groups[fileName]) { @@ -64,7 +65,7 @@ const internalOpenappsecLog = { eventSeverity: json.eventSeverity, eventLevel: json.eventLevel, eventTime: json.eventTime, - eventName: json.eventName + eventName: json.eventName }; wrappedObjects.push(wrappedObject); } catch (err) { @@ -83,7 +84,139 @@ const internalOpenappsecLog = { let groupedFiles = listLogFiles(directoryPath).catch(console.error); return groupedFiles; }); - } + }, + + countTotalLines: async function (directoryPath) { + const files = await fs.promises.readdir(directoryPath); + const logFiles = files.filter(file => path.extname(file).startsWith('.log')); + + let totalLineCount = 0; + + for (const file of logFiles) { + const filePath = path.join(directoryPath, file); + + // Read only the first line of the file + const readStream = fs.createReadStream(filePath); + const rl = readline.createInterface({ input: readStream }); + const firstLine = await new Promise(resolve => { + rl.on('line', line => { + rl.close(); + resolve(line); + }); + }); + + // Check if the first line is a non-data line + try { + JSON.parse(firstLine); + } catch (err) { + continue; // Skip this file if the first line is a non-data line + } + + // If the first line is a data line, read the rest of the file + const content = await fs.promises.readFile(filePath, 'utf8'); + const lines = content.split('\n'); + totalLineCount += lines.length; + } + + return totalLineCount; + }, + + processFile: async function (filePath) { + const content = await fs.promises.readFile(filePath, 'utf8'); + const lines = content.split('\n'); + const dataLines = []; + + for (const line of lines) { + try { + const json = JSON.parse(line); + const groupName = path.basename(filePath, path.extname(filePath)); + const wrappedObject = { + source: groupName, + meta: json, + serviceName: json.eventSource.serviceName, + eventPriority: json.eventPriority, + eventSeverity: json.eventSeverity, + eventLevel: json.eventLevel, + eventTime: json.eventTime, + eventName: json.eventName + }; + dataLines.push(wrappedObject); + } catch (err) { + // Ignore lines that don't contain JSON data + } + } + + return dataLines; + }, + + + getAll: function (access, expand, search_query) { + return access.can('auditlog:list') + .then(async () => { + const directoryPath = '/app/openappsec_files/logs'; + const files = await fs.promises.readdir(directoryPath); + const logFiles = files.filter(file => path.extname(file).startsWith('.log')); + + // Sort the logFiles array + logFiles.sort((a, b) => { + const baseA = path.basename(a, path.extname(a)); + const baseB = path.basename(b, path.extname(b)); + return baseA.localeCompare(baseB, undefined, { numeric: true, sensitivity: 'base' }); + }); + + const wrappedObjects = []; + for (const file of logFiles) { + const filePath = path.join(directoryPath, file); + const dataLines = await this.processFile(filePath); + wrappedObjects.push(...dataLines); + } + + return wrappedObjects; + }); + }, + + + getPage: function (access, expand, search_query, page, perPage) { + return access.can('auditlog:list') + .then(async () => { + const directoryPath = '/app/openappsec_files/logs'; + + let totalDataLines = await this.countTotalLines(directoryPath); + console.log("totalLineCount: " + totalDataLines); + + const files = await fs.promises.readdir(directoryPath); + const logFiles = files.filter(file => path.extname(file).startsWith('.log')); + + // Sort the logFiles array + logFiles.sort((a, b) => { + const baseA = path.basename(a, path.extname(a)); + const baseB = path.basename(b, path.extname(b)); + return baseA.localeCompare(baseB, undefined, { numeric: true, sensitivity: 'base' }); + }); + + const wrappedObjects = []; + let lineCount = 0; + let start = (page - 1) * perPage; + let end = page * perPage; + + for (const file of logFiles) { + if (lineCount >= end) { + break; + } + + const filePath = path.join(directoryPath, file); + const dataLines = await this.processFile(filePath); + const pageDataLines = dataLines.slice(start - lineCount, end - lineCount); + wrappedObjects.push(...pageDataLines); + lineCount += pageDataLines.length; + } + + return { + data: wrappedObjects, + totalDataLines: totalDataLines, + }; + }); + }, }; module.exports = internalOpenappsecLog; diff --git a/frontend/js/app/api.js b/frontend/js/app/api.js index 2c198bf5..6829745a 100644 --- a/frontend/js/app/api.js +++ b/frontend/js/app/api.js @@ -112,7 +112,7 @@ function makeExpansionString(expand) { * @param {String} [query] * @returns {Promise} */ -function getAllObjects(path, expand, query) { +function getAllObjects(path, expand, query, page, perPage) { let params = []; if (typeof expand === 'object' && expand !== null && expand.length) { @@ -123,7 +123,13 @@ function getAllObjects(path, expand, query) { params.push('query=' + query); } - return fetch('get', path + (params.length ? '?' + params.join('&') : '')); + if (page && perPage) { + params.push('page=' + page); + params.push('perPage=' + perPage); + } + + let url = path + (params.length ? '?' + params.join('&') : ''); + return fetch('get', url); } function FileUpload(path, fd) { @@ -724,7 +730,7 @@ module.exports = { */ getAll: function (expand, query) { return getAllObjects('openappsec-log', expand, query); - } + }, }, Reports: { diff --git a/frontend/js/app/controller.js b/frontend/js/app/controller.js index eb9677bb..3bb6a94d 100644 --- a/frontend/js/app/controller.js +++ b/frontend/js/app/controller.js @@ -410,12 +410,18 @@ module.exports = { /** * openappsec Log */ - showOpenappsecLog: function () { + showOpenappsecLogPage: function (page) { + page = parseInt(page) || 1; let controller = this; if (Cache.User.isAdmin()) { require(['./main', './openappsec-log/main'], (App, View) => { - controller.navigate('/openappsec-log'); - App.UI.showAppContent(new View()); + controller.navigate('/openappsec-log/page/' + page); + + // Show the view with the data + App.UI.showAppContent(new View({ + page: page, + perPage: 50 + })); }); } else { this.showDashboard(); diff --git a/frontend/js/app/openappsec-log/list/main.js b/frontend/js/app/openappsec-log/list/main.js index 9d3e26fb..01f5edd4 100755 --- a/frontend/js/app/openappsec-log/list/main.js +++ b/frontend/js/app/openappsec-log/list/main.js @@ -2,9 +2,28 @@ const Mn = require('backbone.marionette'); const ItemView = require('./item'); const template = require('./main.ejs'); -const TableBody = Mn.CollectionView.extend({ - tagName: 'tbody', - childView: ItemView +let TableBody = Mn.CollectionView.extend({ + tagName: 'tbody', + childView: ItemView, + + initialize: function (options) { + this.options = new Backbone.Model(options); + this.page = options.page; + this.perPage = options.perPage; + this.updatePage(); + this.listenTo(this.options, 'change:page', this.updatePage); + }, + + setPage: function (page) { + this.page = page; + this.updatePage(); + this.render(); + }, + + updatePage: function () { + let models = this.collection.models.slice((this.page - 1) * this.perPage, this.page * this.perPage); + this.collection.reset(models); + } }); module.exports = Mn.View.extend({ @@ -21,7 +40,9 @@ module.exports = Mn.View.extend({ onRender: function () { this.showChildView('body', new TableBody({ - collection: this.collection + collection: this.collection, + page: this.options.page, + perPage: this.options.perPage })); } }); diff --git a/frontend/js/app/openappsec-log/main.ejs b/frontend/js/app/openappsec-log/main.ejs index ed36305a..81c0cabc 100755 --- a/frontend/js/app/openappsec-log/main.ejs +++ b/frontend/js/app/openappsec-log/main.ejs @@ -3,14 +3,14 @@