Merge branch 'client-side-log-pagination' into open-appsec-integration

backend/internal/openappsec-log.js

@@ -14,7 +14,7 @@ const internalOpenappsecLog = {
 	 * @param   {String}  [search_query]
 	 * @returns {Promise}
 	 */
-	getAll: (access, expand, search_query) => {
+	getAllold: (access, expand, search_query) => {
 		return access.can('auditlog:list')
 			.then(() => {
 
@@ -37,6 +37,7 @@ const internalOpenappsecLog = {
 						return path.extname(a).localeCompare(path.extname(b));
 					});
 
+					// Group the log files by their base name
 					const groupedFiles = sortedLogFiles.reduce((groups, file) => {
 						const fileName = path.basename(file, path.extname(file));
 						if (!groups[fileName]) {
@@ -64,7 +65,7 @@ const internalOpenappsecLog = {
 											eventSeverity: json.eventSeverity,
 											eventLevel: json.eventLevel,
 											eventTime: json.eventTime,
-              								eventName: json.eventName
+											eventName: json.eventName
 										};
 										wrappedObjects.push(wrappedObject);
 									} catch (err) {
@@ -83,7 +84,139 @@ const internalOpenappsecLog = {
 				let groupedFiles = listLogFiles(directoryPath).catch(console.error);
 				return groupedFiles;
 			});
-	}
+	},
+
+	countTotalLines: async function (directoryPath) {
+		const files = await fs.promises.readdir(directoryPath);
+		const logFiles = files.filter(file => path.extname(file).startsWith('.log'));
+
+		let totalLineCount = 0;
+
+		for (const file of logFiles) {
+			const filePath = path.join(directoryPath, file);
+
+			// Read only the first line of the file
+			const readStream = fs.createReadStream(filePath);
+			const rl = readline.createInterface({ input: readStream });
+			const firstLine = await new Promise(resolve => {
+				rl.on('line', line => {
+					rl.close();
+					resolve(line);
+				});
+			});
+
+			// Check if the first line is a non-data line
+			try {
+				JSON.parse(firstLine);
+			} catch (err) {
+				continue; // Skip this file if the first line is a non-data line
+			}
+
+			// If the first line is a data line, read the rest of the file
+			const content = await fs.promises.readFile(filePath, 'utf8');
+			const lines = content.split('\n');
+			totalLineCount += lines.length;
+		}
+
+		return totalLineCount;
+	},
+
+	processFile: async function (filePath) {
+		const content = await fs.promises.readFile(filePath, 'utf8');
+		const lines = content.split('\n');
+		const dataLines = [];
+
+		for (const line of lines) {
+			try {
+				const json = JSON.parse(line);
+				const groupName = path.basename(filePath, path.extname(filePath));
+				const wrappedObject = {
+					source: groupName,
+					meta: json,
+					serviceName: json.eventSource.serviceName,
+					eventPriority: json.eventPriority,
+					eventSeverity: json.eventSeverity,
+					eventLevel: json.eventLevel,
+					eventTime: json.eventTime,
+					eventName: json.eventName
+				};
+				dataLines.push(wrappedObject);
+			} catch (err) {
+				// Ignore lines that don't contain JSON data
+			}
+		}
+
+		return dataLines;
+	},
+
+
+	getAll: function (access, expand, search_query) {
+		return access.can('auditlog:list')
+			.then(async () => {
+				const directoryPath = '/app/openappsec_files/logs';
+				const files = await fs.promises.readdir(directoryPath);
+				const logFiles = files.filter(file => path.extname(file).startsWith('.log'));
+
+				// Sort the logFiles array
+				logFiles.sort((a, b) => {
+					const baseA = path.basename(a, path.extname(a));
+					const baseB = path.basename(b, path.extname(b));
+					return baseA.localeCompare(baseB, undefined, { numeric: true, sensitivity: 'base' });
+				});
+
+				const wrappedObjects = [];
+				for (const file of logFiles) {
+					const filePath = path.join(directoryPath, file);
+					const dataLines = await this.processFile(filePath);
+					wrappedObjects.push(...dataLines);
+				}
+
+				return wrappedObjects;
+			});
+	},
+
+
+	getPage: function (access, expand, search_query, page, perPage) {
+		return access.can('auditlog:list')
+			.then(async () => {
+				const directoryPath = '/app/openappsec_files/logs';
+
+				let totalDataLines = await this.countTotalLines(directoryPath);
+				console.log("totalLineCount: " + totalDataLines);
+
+				const files = await fs.promises.readdir(directoryPath);
+				const logFiles = files.filter(file => path.extname(file).startsWith('.log'));
+
+				// Sort the logFiles array
+				logFiles.sort((a, b) => {
+					const baseA = path.basename(a, path.extname(a));
+					const baseB = path.basename(b, path.extname(b));
+					return baseA.localeCompare(baseB, undefined, { numeric: true, sensitivity: 'base' });
+				});
+
+				const wrappedObjects = [];
+				let lineCount = 0;
+				let start = (page - 1) * perPage;
+				let end = page * perPage;
+
+				for (const file of logFiles) {
+					if (lineCount >= end) {
+						break;
+					}
+
+					const filePath = path.join(directoryPath, file);
+					const dataLines = await this.processFile(filePath);
+					const pageDataLines = dataLines.slice(start - lineCount, end - lineCount);
+					wrappedObjects.push(...pageDataLines);
+					lineCount += pageDataLines.length;
+				}
+
+				return {
+					data: wrappedObjects,
+					totalDataLines: totalDataLines,
+				};
+			});
+	},
 };
 
 module.exports = internalOpenappsecLog;

frontend/js/app/api.js

@@ -112,7 +112,7 @@ function makeExpansionString(expand) {
  * @param   {String}   [query]
  * @returns {Promise}
  */
-function getAllObjects(path, expand, query) {
+function getAllObjects(path, expand, query, page, perPage) {
     let params = [];
 
     if (typeof expand === 'object' && expand !== null && expand.length) {
@@ -123,7 +123,13 @@ function getAllObjects(path, expand, query) {
         params.push('query=' + query);
     }
 
-    return fetch('get', path + (params.length ? '?' + params.join('&') : ''));
+    if (page && perPage) {
+        params.push('page=' + page);
+        params.push('perPage=' + perPage);
+    }
+
+    let url = path + (params.length ? '?' + params.join('&') : '');
+    return fetch('get', url);
 }
 
 function FileUpload(path, fd) {
@@ -724,7 +730,7 @@ module.exports = {
          */
         getAll: function (expand, query) {
             return getAllObjects('openappsec-log', expand, query);
-        }
+        },
     },
 
     Reports: {

frontend/js/app/controller.js

@@ -410,12 +410,18 @@ module.exports = {
     /**
      * openappsec Log
      */
-    showOpenappsecLog: function () {
+    showOpenappsecLogPage: function (page) {
+        page = parseInt(page) || 1;
         let controller = this;
         if (Cache.User.isAdmin()) {
             require(['./main', './openappsec-log/main'], (App, View) => {
-                controller.navigate('/openappsec-log');
+                controller.navigate('/openappsec-log/page/' + page);
-                App.UI.showAppContent(new View());
+
+                // Show the view with the data
+                App.UI.showAppContent(new View({
+                    page: page,
+                    perPage: 50
+                }));
             });
         } else {
             this.showDashboard();

frontend/js/app/openappsec-log/list/main.js

@@ -2,9 +2,28 @@ const Mn       = require('backbone.marionette');
 const ItemView = require('./item');
 const template = require('./main.ejs');
 
-const TableBody = Mn.CollectionView.extend({
+let TableBody = Mn.CollectionView.extend({
-    tagName:   'tbody',
+    tagName: 'tbody',
-    childView: ItemView
+    childView: ItemView,
+
+    initialize: function (options) {
+        this.options = new Backbone.Model(options);
+        this.page = options.page;
+        this.perPage = options.perPage;
+        this.updatePage();
+        this.listenTo(this.options, 'change:page', this.updatePage);
+    },
+
+    setPage: function (page) {
+        this.page = page;
+        this.updatePage();
+        this.render();
+    },
+
+    updatePage: function () {
+        let models = this.collection.models.slice((this.page - 1) * this.perPage, this.page * this.perPage);
+        this.collection.reset(models);
+    }
 });
 
 module.exports = Mn.View.extend({
@@ -21,7 +40,9 @@ module.exports = Mn.View.extend({
 
     onRender: function () {
         this.showChildView('body', new TableBody({
-            collection: this.collection
+            collection: this.collection,
+            page:       this.options.page,
+            perPage:    this.options.perPage
         }));
     }
 });

frontend/js/app/openappsec-log/main.ejs

@@ -3,14 +3,14 @@
     <div class="card-header">
         <h3 class="card-title">Security Log</h3>
         <div class="card-options">
-            <form class="search-form" role="search">
+            <!-- <form class="search-form" role="search">
                 <div class="input-icon">
                     <span class="input-icon-addon">
                       <i class="fe fe-search"></i>
                     </span>
                     <input name="source-query" type="text" value="" class="form-control form-control-sm" placeholder="<%- i18n('audit-log', 'search') %>" aria-label="<%- i18n('audit-log', 'search') %>">
                 </div>
-            </form>
+            </form> -->
         </div>
     </div>
     <div class="card-body no-padding min-100">
@@ -19,6 +19,9 @@
             <div class="dimmer-content list-region">
                 <!-- List Region -->
             </div>
+            <div class="dimmer-content pagination-region">
+                <!-- Pagination Region -->
+            </div>
         </div>
 
     </div>

frontend/js/app/openappsec-log/main.js

@@ -1,35 +1,99 @@
-const Mn            = require('backbone.marionette');
+const Mn = require('backbone.marionette');
-const App           = require('../main');
+const App = require('../main');
 const OpenappsecLogModel = require('../../models/openappsec-log');
-const ListView      = require('./list/main');
+const ListView = require('./list/main');
-const template      = require('./main.ejs');
+const template = require('./main.ejs');
-const ErrorView     = require('../error/main');
+const ErrorView = require('../error/main');
-const EmptyView     = require('../empty/main');
+const EmptyView = require('../empty/main');
+const { data } = require('jquery');
+const Controller = require('../controller');
+
+let PaginationView = Mn.View.extend({
+    tagName: 'ul',
+    className: 'pagination justify-content-center mt-4 border-top pt-3',
+
+    initialize: function (options) {
+        this.totalPages = parseInt(options.totalPages) || 1;
+        this.currentPage = parseInt(options.currentPage) || 1;
+        this.totalDataLines = parseInt(options.totalDataLines) || 1;
+        this.maxPageLinks = 15;
+    },
+
+    template: _.template(
+        '<li class="page-item <%= currentPage === 1 ? "disabled" : "" %>">' +
+        '<a class="page-link" href="/openappsec-log/page/1">First</a>' +
+        '</li>' +
+        '<li class="page-item <%= currentPage === 1 ? "disabled" : "" %>">' +
+        '<a class="page-link" href="/openappsec-log/page/<%= currentPage - 1 %>">Previous</a>' +
+        '</li>' +
+        '<% let startPage = Math.max(1, currentPage - Math.floor(maxPageLinks / 2)); %>' +
+        '<% let endPage = Math.min(startPage + maxPageLinks - 1, totalPages); %>' +
+        '<% startPage = Math.max(1, endPage - maxPageLinks + 1); %>' +
+        '<% for (let i = startPage; i <= endPage; i++) { %>' +
+        '<li class="page-item <%= i === currentPage ? "active" : "" %>">' +
+        '<a class="page-link" href="/openappsec-log/page/<%= i %>"><%= i %></a>' +
+        '</li>' +
+        '<% } %>' +
+        '<li class="page-item <%= currentPage === totalPages ? "disabled" : "" %>">' +
+        '<a class="page-link" href="/openappsec-log/page/<%= currentPage + 1 %>">Next</a>' +
+        '</li>' +
+        '<li class="page-item <%= currentPage === totalPages ? "disabled" : "" %>">' +
+        '<a class="page-link" href="/openappsec-log/page/<%= totalPages %>">Last</a>' +
+        '</li>' +
+        '<li class="page-item disabled">' +
+        '<span class="page-link">Total lines: <%= totalDataLines %></span>' +
+        '</li>'
+    ),
+
+    templateContext: function () {
+        return {
+            totalDataLines: this.totalDataLines,
+            totalPages: this.totalPages,
+            currentPage: this.currentPage,
+            maxPageLinks: this.maxPageLinks
+        };
+    }
+});
 
 module.exports = Mn.View.extend({
-    id:       'openappsec-log',
+    id: 'openappsec-log',
     template: template,
+    
+    initialize: function (options) {
+        this.options = options;
+        // this.listenTo(App, 'openappsec-log:page', this.setPage);
+    },    
 
     ui: {
         list_region: '.list-region',
-        dimmer:      '.dimmer',
+        dimmer: '.dimmer',
-        search:      '.search-form',
+        search: '.search-form',
-        query:       'input[name="source-query"]'
+        query: 'input[name="source-query"]',
+        pagination_region: '.pagination-region'
     },
 
     fetch: App.Api.OpenappsecLog.getAll,
 
-    showData: function(response) {
+    showData: function (response) {
+        const totalDataLines = response.length;
         this.showChildView('list_region', new ListView({
-            collection: new OpenappsecLogModel.Collection(response)
+            collection: new OpenappsecLogModel.Collection(response),
+            page: this.options.page,
+            perPage: this.options.perPage
+        }));
+
+        this.showChildView('pagination_region', new PaginationView({
+            totalDataLines: totalDataLines,
+            totalPages: Math.ceil(totalDataLines / this.options.perPage),
+            currentPage: this.options.page
         }));
     },
 
-    showError: function(err) {
+    showError: function (err) {
         this.showChildView('list_region', new ErrorView({
-            code:    err.code,
+            code: err.code,
             message: err.message,
-            retry:   function () {
+            retry: function () {
                 App.Controller.showOpenappsecLog();
             }
         }));
@@ -37,15 +101,16 @@ module.exports = Mn.View.extend({
         console.error(err);
     },
 
-    showEmpty: function() {
+    showEmpty: function () {
         this.showChildView('list_region', new EmptyView({
-            title:    App.i18n('audit-log', 'empty'),
+            title: App.i18n('audit-log', 'empty'),
             subtitle: App.i18n('audit-log', 'empty-subtitle')
         }));
     },
 
     regions: {
-        list_region: '@ui.list_region'
+        list_region: '@ui.list_region',
+        pagination_region: '@ui.pagination_region'
     },
 
     events: {
@@ -58,15 +123,24 @@ module.exports = Mn.View.extend({
                 .catch(err => {
                     this.showError(err);
                 });
+        },
+
+        'click @ui.pagination_region a': function (e) {
+            e.preventDefault();
+
+            // get the page number from the link
+            const newPage = $(e.currentTarget).attr('href').split('/').pop();
+            Controller.navigate('/openappsec-log/page/' + newPage, { trigger: true });
         }
     },
 
     onRender: function () {
         let view = this;
+        let query = this.ui.query.val() || '';
 
-        view.fetch(['user'])
+        view.fetch(['user'], query)
             .then(response => {
-                if (!view.isDestroyed() && response && response.length) {
+                if (!view.isDestroyed() && response) {
                     view.showData(response);
                 } else {
                     view.showEmpty();

frontend/js/app/router.js

@@ -13,7 +13,8 @@ module.exports = AppRouter.default.extend({
         'nginx/access':       'showNginxAccess',
         'nginx/certificates': 'showNginxCertificates',
         'audit-log':          'showAuditLog',
-        'openappsec-log':     'showOpenappsecLog',
+        'openappsec-log':     'showOpenappsecLogPage',
+        'openappsec-log/page/:number':     'showOpenappsecLogPage',
         'settings':           'showSettings',
         '*default':           'showDashboard'
     }