Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 65 additions & 0 deletions arangod/RestHandler/RestAdminLogHandler.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@
#include "Logger/Logger.h"
#include "Logger/LoggerFeature.h"
#include "Logger/LogTopic.h"
#include "Logger/LogLevel.h"
#include "Logger/LogMacros.h"
#include "Network/Methods.h"
#include "Network/NetworkFeature.h"
#include "Network/Utils.h"
Expand Down Expand Up @@ -143,6 +145,8 @@ auto RestAdminLogHandler::executeAsync() -> futures::Future<futures::Unit> {
"where suffix can be either 'level' or 'structured'");
}
}
} else if (type == rest::RequestType::POST && suffixes.size() == 0) {
co_await handleLogWrite();
} else {
generateError(rest::ResponseCode::METHOD_NOT_ALLOWED,
TRI_ERROR_HTTP_METHOD_NOT_ALLOWED);
Expand Down Expand Up @@ -590,6 +594,67 @@ auto RestAdminLogHandler::handleLogLevel() -> async<void> {
}
}

auto RestAdminLogHandler::handleLogWrite() -> async<void> {
bool parseSuccess = false;
VPackSlice slice = this->parseVPackBody(parseSuccess);
if (!parseSuccess) {
co_return;
}
parseSuccess = false;
if (slice.isArray()) {
for (VPackSlice logLine : VPackArrayIterator(slice)) {
auto logId = logLine.get("ID").stringView();
auto logLevel = logLine.get("level").stringView();
std::string prefix;
if (logLevel.compare("fatal") == 0) {
prefix = "FATAL! ";
} else if (logLevel.compare("error") != 0 &&
logLevel.compare("warning") != 0 &&
logLevel.compare("warn") != 0 &&
logLevel.compare("info") != 0 &&
logLevel.compare("debug") != 0 &&
logLevel.compare("trace") != 0) {
// invalid log level
prefix = logLevel;
prefix += "!";
}
auto logTopic = logLine.get("topic").stringView();
auto logMessage = logLine.get("message").stringView();
LogTopic const* topicPtr =
logTopic.empty() ? nullptr : LogTopic::lookup(logTopic);
LogTopic const& topic = (topicPtr != nullptr) ? *topicPtr : Logger::FIXME;
auto logMessageS = [&](auto const& message, auto const& logId) {
if (logLevel.compare("fatal") == 0) {
LOG_TOPIC(logId, FATAL, topic) << prefix << message;
} else if (logLevel.compare("error") == 0) {
LOG_TOPIC(logId, ERR, topic) << prefix << message;
} else if (logLevel.compare("warning") == 0 ||
logLevel.compare("warn") == 0) {
LOG_TOPIC(logId, WARN, topic) << prefix << message;
} else if (logLevel.compare("info") == 0) {
LOG_TOPIC(logId, INFO, topic) << prefix << message;
} else if (logLevel.compare("debug") == 0) {
LOG_TOPIC(logId, DEBUG, topic) << prefix << message;
} else if (logLevel.compare("trace") == 0) {
LOG_TOPIC(logId, TRACE, topic) << prefix << message;
} else {
LOG_TOPIC(logId, WARN, topic) << prefix << message;
}
};
logMessageS(logMessage, logId);
}
parseSuccess = true;
}
if (!parseSuccess) {
generateError(rest::ResponseCode::BAD, TRI_ERROR_HTTP_BAD_PARAMETER);
} else {
velocypack::Builder body;

generateResult(rest::ResponseCode::ACCEPTED, body.slice());
}
co_return;
}

void RestAdminLogHandler::handleLogStructuredParams() {
auto const type = _request->requestType();

Expand Down
12 changes: 12 additions & 0 deletions arangod/RestHandler/RestAdminLogHandler.h
Original file line number Diff line number Diff line change
Expand Up @@ -41,11 +41,23 @@ class RestAdminLogHandler : public RestBaseHandler {
RequestLane lane() const override final { return RequestLane::CLIENT_FAST; }
auto executeAsync() -> futures::Future<futures::Unit> override;

protected:
// we just use the database from the URL to log it.
[[nodiscard]] auto makeSharedLogContextValue() const
-> std::shared_ptr<LogContext::Values> override {
return LogContext::makeValue()
.with<structuredParams::UrlName>(_request->fullUrl())
.with<structuredParams::UserName>(_request->user())
.with<structuredParams::DatabaseName>(_request->databaseName())
.share();
}

private:
arangodb::Result verifyPermitted();
void clearLogs();
auto reportLogs(bool newFormat) -> async<void>;
auto handleLogLevel() -> async<void>;
auto handleLogWrite() -> async<void>;
void handleLogStructuredParams();
};
} // namespace arangodb
29 changes: 22 additions & 7 deletions js/client/modules/@arangodb/test-helper.js
Original file line number Diff line number Diff line change
Expand Up @@ -804,7 +804,7 @@ exports.AQL_EXECUTE = function(query, bindVars, options) {
};

exports.insertManyDocumentsIntoCollection
= function(db, coll, maker, limit, batchSize) {
= function(db, coll, maker, limit, batchSize, abortFunc = () => false) {
// This function uses the asynchronous API of `arangod` to quickly
// insert a lot of documents into a collection. You can control which
// documents to insert with the `maker` function. The arguments are:
Expand Down Expand Up @@ -862,14 +862,20 @@ exports.insertManyDocumentsIntoCollection
l = [];
}
let i = 0;
while (i < jobs.length) {
let r = arango.PUT_RAW(`/_api/job/${jobs[i]}`, {});
if (r.code === 204) {
i += 1;
} else if (r.code === 202) {
jobs = jobs.slice(0, i).concat(jobs.slice(i+1));
if (jobs.length > 10 || done) {
while (i < jobs.length) {
let r = arango.PUT_RAW(`/_api/job/${jobs[i]}`, {});
if (r.code === 204) {
i += 1;
} else if (r.code === 202) {
jobs = jobs.slice(0, i).concat(jobs.slice(i+1));
}
}
}
if (abortFunc()) {
print('aborting insert loop by hook');
return;
}
if (done) {
if (jobs.length === 0) {
break;
Expand All @@ -879,6 +885,15 @@ exports.insertManyDocumentsIntoCollection
}
};

exports.logServer = function (message, level='info', ID="aaaaa", topic='general') {
return arango.POST_RAW('/_admin/log/', [{
level,
ID,
topic,
message
}]);
};

exports.executeExternalAndWaitWithSanitizer = function (executable, args, tmpFileName, options = global.instanceManager.options) {
let sanHnd = new sanHandler(executable, options);
let tmpMgr = new tmpDirMngr(fs.join(tmpFileName), options);
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*jshint globalstrict:false, strict:false */
/* global getOptions, assertTrue, assertFalse, arango, assertEqual */
/* global getOptions, print, assertTrue, assertFalse, arango, assertEqual */

// //////////////////////////////////////////////////////////////////////////////
// / DISCLAIMER
Expand Down Expand Up @@ -28,7 +28,6 @@
if (getOptions === true) {
return {
'log.max-queued-entries': '0',
'javascript.allow-admin-execute': 'true',
'log.force-direct': 'false',
'log.foreground-tty': 'false',
};
Expand All @@ -52,8 +51,16 @@ function LoggerSuite() {
arango.PUT("/_admin/log/level", { general: "info" });

// messages with level "info" will be dropped
let res = arango.POST_RAW("/_admin/execute", "for (let i = 0; i < 100; ++i) { require('console').log('abc'); }");
assertEqual(200, res.code, res);
let messages = [];
for (let i = 0; i < 100; ++i) {
messages.push({
level: "info",
ID: "aaaaa",
topic: "general",
message: `abc ${i}`
});
}
arango.POST_RAW('/_admin/log/', messages);

const newValue = getMetric("arangodb_logger_messages_dropped_total");
assertTrue(newValue >= oldValue + 100, {oldValue, newValue});
Expand All @@ -72,8 +79,16 @@ function LoggerSuite() {
arango.PUT("/_admin/log/level", { general: "warn" });

// messages with level "warn" will not be dropped
let res = arango.POST_RAW("/_admin/execute", "for (let i = 0; i < 100; ++i) { require('console').warn('abc'); }");
assertEqual(200, res.code, res);
let messages = [];
for (let i = 0; i < 100; ++i) {
messages.push({
level: "warn",
ID: "bbbbb",
topic: "general",
message: `abc ${i}`
});
}
arango.POST_RAW('/_admin/log/', messages);

const newValue = getMetric("arangodb_logger_messages_dropped_total");
// allow for up to 20 unrelated log messages (from other server
Expand All @@ -89,8 +104,16 @@ function LoggerSuite() {
const oldValue = getMetric("arangodb_logger_messages_dropped_total");

// messages with level "error" will not be dropped
let res = arango.POST_RAW("/_admin/execute", "for (let i = 0; i < 100; ++i) { require('console').error('abc'); }");
assertEqual(200, res.code, res);
let messages = [];
for (let i = 0; i < 100; ++i) {
messages.push({
level: "error",
ID: "ccccc",
topic: "general",
message: `abc ${i}`
});
}
arango.POST_RAW('/_admin/log/', messages);

const newValue = getMetric("arangodb_logger_messages_dropped_total");
// allow for up to 20 unrelated log messages (from other server
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*jshint globalstrict:false, strict:false */
/* global getOptions, assertTrue, arango, assertEqual, assertMatch */
/* global GLOBAL, print, getOptions, assertTrue, arango, assertEqual, assertMatch */

// //////////////////////////////////////////////////////////////////////////////
// / DISCLAIMER
Expand All @@ -24,16 +24,13 @@
/// @author Julia Puget
// //////////////////////////////////////////////////////////////////////////////

const fs = require('fs');

if (getOptions === true) {
return {
'log.hostname': 'delorean',
'log.process': 'false',
'log.ids': 'false',
'log.role': 'true',
'log.thread': 'true',
'log.output': 'file://' + fs.getTempFile() + '.$PID',
'log.foreground-tty': 'false',
'log.level': 'debug',
'log.escape-unicode-chars': 'false',
Expand All @@ -42,56 +39,56 @@ if (getOptions === true) {
};
}

const fs = require('fs');
const jsunity = require('jsunity');
const { logServer } = require('@arangodb/test-helper');
const IM = GLOBAL.instanceManager;

function EscapeControlFalseSuite() {
'use strict';

return {
testEscapeControlFalse: function() {
const controlCharsLength = 31;
const request = `require('console').log("testmann: start");
IM.rememberConnection();
IM.arangods.forEach(arangod => {
print(`testing ${arangod.name}`);
arangod.connect();
const controlCharsLength = 31;
logServer("testmann: start");
for (let i = 1; i <= 31; ++i) {
let controlChar = '"\\\\u' + i.toString(16).padStart(4, '0') + '"';
let controlChar = '"\\u' + i.toString(16).padStart(4, '0') + '"';
controlChar = JSON.parse(controlChar);
require('console').log("testmann: testi \\u0008\\u0009\\u000A\\u000B\\u000C" + controlChar + " \\u00B0\\ud83e\\uddd9\\uf0f9\\u9095\\uf0f9\\u90b6abc123");
logServer("testmann: testi \u0008\u0009\u000A\u000B\u000C" + controlChar + "\u00B0\ud83e\uddd9\uf0f9\u9095\uf0f9\u90b6abc123");
}
require('console').log("testmann: done");
return require('internal').options()["log.output"];`;

const res = arango.POST("/_admin/execute", request);

assertTrue(Array.isArray(res));
assertTrue(res.length > 0);

let logfile = res[res.length - 1].replace(/^file:\/\//, '');

// log is buffered, so give it a few tries until the log messages appear
let tries = 0;
let filtered = [];
while (++tries < 60) {
let content = fs.readFileSync(logfile, 'utf-8');
let lines = content.split('\n');

filtered = lines.filter((line) => {
return line.match(/testmann: /);
});

if (filtered.length === controlCharsLength + 2) {
break;
logServer("testmann: done", "error"); // error flushes
// log is buffered, so give it a few tries until the log messages appear
let tries = 0;
let filtered = [];
while (++tries < 60) {
let content = fs.readFileSync(arangod.logFile, 'utf-8');
let lines = content.split('\n');

filtered = lines.filter((line) => {
return line.match(/testmann: /);
});

if (filtered.length === controlCharsLength + 2) {
break;
}

require("internal").sleep(0.5);
}

require("internal").sleep(0.5);
}

assertEqual(controlCharsLength + 2, filtered.length);
assertMatch(/testmann: start/, filtered[0]);
for (let i = 1; i < controlCharsLength + 1; ++i) {
const parsedRes = JSON.parse(filtered[i]);
assertTrue(parsedRes.hasOwnProperty("message"));
assertEqual(parsedRes.message, "testmann: testi \u00B0\ud83e\uddd9\uf0f9\u9095\uf0f9\u90b6abc123");
}
assertMatch(/testmann: done/, filtered[controlCharsLength + 1]);
assertEqual(controlCharsLength + 2, filtered.length);
assertMatch(/testmann: start/, filtered[0]);
for (let i = 1; i < controlCharsLength + 1; ++i) {
const parsedRes = JSON.parse(filtered[i]);
assertTrue(parsedRes.hasOwnProperty("message"));
assertEqual(parsedRes.message, "testmann: testi \u00B0\ud83e\uddd9\uf0f9\u9095\uf0f9\u90b6abc123");
}
assertMatch(/testmann: done/, filtered[controlCharsLength + 1]);
});
IM.reconnectMe();
},

};
Expand Down
Loading