Skip to content

Commit e5e7eca

Browse files
garryyangarryyan
committed
fix user define xlog crypt error
1 parent e19a1b9 commit e5e7eca

File tree

12 files changed

+374
-330
lines changed

12 files changed

+374
-330
lines changed

‎mars/libraries/mars_android_sdk/jni/decode_mars_log_file.py.rewriteme

Lines changed: 109 additions & 100 deletions
Original file line numberDiff line numberDiff line change
@@ -5,136 +5,145 @@ import os
55
import glob
66
import zlib
77
import struct
8-
8+
#import base64
99

1010
MAGIC_NO_COMPRESS_START = 0x03;
11-
MAGIC_COMPRESS_START = 0x04;
11+
MAGIC_COMPRESS_START = 0x05;
1212

1313
MAGIC_END = 0x00;
1414

1515
lastseq = 0;
1616

1717
def IsGoodLogBuffer(_buffer, _offset, count):
1818

19-
if _offset == len(_buffer): return (True, '')
19+
if _offset == len(_buffer): return (True, '')
2020

21-
if MAGIC_NO_COMPRESS_START==_buffer[_offset] or MAGIC_COMPRESS_START==_buffer[_offset]:
22-
headerLen = 1 + 2 + 1 + 1 + 4 + 4
23-
else:
24-
return (False, '_buffer[%d]:%d != MAGIC_NUM_START'%(_offset, _buffer[_offset]))
21+
if MAGIC_NO_COMPRESS_START==_buffer[_offset] or MAGIC_COMPRESS_START==_buffer[_offset]:
22+
headerLen = 1 + 2 + 1 + 1 + 4 + 4
23+
else:
24+
return (False, '_buffer[%d]:%d != MAGIC_NUM_START'%(_offset, _buffer[_offset]))
2525

2626

27-
if _offset + headerLen + 1 + 1 > len(_buffer): return (False, 'offset:%d > len(buffer):%d'%(_offset, len(_buffer)))
28-
length = struct.unpack_from("I", buffer(_buffer, _offset+headerLen-4-4, 4))[0]
29-
if _offset + headerLen + length + 1 > len(_buffer): return (False, 'log length:%d, end pos %d > len(buffer):%d'%(length, _offset + headerLen + length + 1, len(_buffer)))
30-
if MAGIC_END!=_buffer[_offset + headerLen + length]: return (False, 'log length:%d, buffer[%d]:%d != MAGIC_END'%(length, _offset + headerLen + length, _buffer[_offset + headerLen + length]))
27+
if _offset + headerLen + 1 + 1 > len(_buffer): return (False, 'offset:%d > len(buffer):%d'%(_offset, len(_buffer)))
28+
length = struct.unpack_from("I", buffer(_buffer, _offset+headerLen-4-4, 4))[0]
29+
if _offset + headerLen + length + 1 > len(_buffer): return (False, 'log length:%d, end pos %d > len(buffer):%d'%(length, _offset + headerLen + length + 1, len(_buffer)))
30+
if MAGIC_END!=_buffer[_offset + headerLen + length]: return (False, 'log length:%d, buffer[%d]:%d != MAGIC_END'%(length, _offset + headerLen + length, _buffer[_offset + headerLen + length]))
3131

3232

33-
if (1>=count): return (True, '')
34-
else: return IsGoodLogBuffer(_buffer, _offset+headerLen+length+1, count-1)
35-
36-
33+
if (1>=count): return (True, '')
34+
else: return IsGoodLogBuffer(_buffer, _offset+headerLen+length+1, count-1)
35+
36+
3737
def GetLogStartPos(_buffer, _count):
38-
offset = 0
39-
while True:
40-
if offset >= len(_buffer) : break
41-
42-
if MAGIC_NO_COMPRESS_START==_buffer[offset] or MAGIC_COMPRESS_START==_buffer[offset]:
43-
if IsGoodLogBuffer(_buffer, offset, _count)[0]: return offset
44-
offset+=1
45-
46-
return -1
47-
38+
offset = 0
39+
while True:
40+
if offset >= len(_buffer) : break
41+
42+
if MAGIC_NO_COMPRESS_START==_buffer[offset] or MAGIC_COMPRESS_START==_buffer[offset]:
43+
if IsGoodLogBuffer(_buffer, offset, _count)[0]: return offset
44+
offset+=1
45+
46+
return -1
47+
4848
def DecodeBuffer(_buffer, _offset, _outbuffer):
49-
50-
if _offset >= len(_buffer): return -1
51-
# if _offset + 1 + 4 + 1 + 1 > len(_buffer): return -1
52-
ret = IsGoodLogBuffer(_buffer, _offset, 1)
53-
if not ret[0]:
54-
fixpos = GetLogStartPos(_buffer[_offset:], 1)
55-
if -1==fixpos:
56-
return -1
57-
else:
58-
_outbuffer.extend("[F]decode_log_file.py decode error len=%d, result:%s \n"%(fixpos, ret[1]))
59-
_offset += fixpos
49+
50+
if _offset >= len(_buffer): return -1
51+
# if _offset + 1 + 4 + 1 + 1 > len(_buffer): return -1
52+
ret = IsGoodLogBuffer(_buffer, _offset, 1)
53+
if not ret[0]:
54+
fixpos = GetLogStartPos(_buffer[_offset:], 1)
55+
if -1==fixpos:
56+
return -1
57+
else:
58+
_outbuffer.extend("[F]decode_log_file.py decode error len=%d, result:%s \n"%(fixpos, ret[1]))
59+
_offset += fixpos
60+
61+
if MAGIC_NO_COMPRESS_START==_buffer[_offset] or MAGIC_COMPRESS_START==_buffer[_offset]:
62+
headerLen = 1 + 2 + 1 + 1 + 4 + 4
63+
else:
64+
_outbuffer.extend('in DecodeBuffer _buffer[%d]:%d != MAGIC_NUM_START'%(_offset, _buffer[_offset]))
65+
return -1
66+
67+
length = struct.unpack_from("I", buffer(_buffer, _offset+headerLen-4-4, 4))[0]
68+
tmpbuffer = bytearray(length)
69+
6070

61-
if MAGIC_NO_COMPRESS_START==_buffer[_offset] or MAGIC_COMPRESS_START==_buffer[_offset]:
62-
headerLen = 1 + 2 + 1 + 1 + 4 + 4
63-
else:
64-
_outbuffer.extend('in DecodeBuffer _buffer[%d]:%d != MAGIC_NUM_START'%(_offset, _buffer[_offset]))
65-
return -1
71+
seq=struct.unpack_from("H", buffer(_buffer, _offset+headerLen-4-4-2-2, 2))[0]
72+
begin_hour=struct.unpack_from("c", buffer(_buffer, _offset+headerLen-4-4-1-1, 1))[0]
73+
end_hour=struct.unpack_from("c", buffer(_buffer, _offset+headerLen-4-4-1, 1))[0]
6674

67-
length = struct.unpack_from("I", buffer(_buffer, _offset+headerLen-4-4, 4))[0]
68-
tmpbuffer = bytearray(length)
75+
global lastseq
76+
if seq != 0 and seq != 1 and lastseq != 0 and seq != (lastseq+1):
77+
_outbuffer.extend("[F]decode_log_file.py log seq:%d-%d is missing\n" %(lastseq+1, seq-1))
6978

79+
if seq != 0:
80+
lastseq = seq
7081

71-
seq=struct.unpack_from("H", buffer(_buffer, _offset+headerLen-4-4-2-2, 2))[0]
72-
begin_hour=struct.unpack_from("c", buffer(_buffer, _offset+headerLen-4-4-1-1, 1))[0]
73-
end_hour=struct.unpack_from("c", buffer(_buffer, _offset+headerLen-4-4-1, 1))[0]
82+
tmpbuffer[:] = _buffer[_offset+headerLen:_offset+headerLen+length]
7483

75-
global lastseq
76-
if seq != 0 and seq != 1 and lastseq != 0 and seq != (lastseq+1):
77-
_outbuffer.extend("[F]decode_log_file.py log seq:%d-%d is missing\n" %(lastseq+1, seq-1))
84+
try:
85+
86+
if MAGIC_COMPRESS_START==_buffer[_offset]:
87+
decompressor = zlib.decompressobj(-zlib.MAX_WBITS)
88+
decompress_data = bytearray()
89+
while len(tmpbuffer) > 0:
90+
single_log_len = struct.unpack_from("H", tmpbuffer[0:2])[0]
91+
#decompress_data.extend(base64.decodestring(tmpbuffer[2:single_log_len+2]))
92+
decompress_data.extend(tmpbuffer[2:single_log_len+2])
93+
tmpbuffer[:] = tmpbuffer[single_log_len+2:len(tmpbuffer)]
7894

79-
if seq != 0:
80-
lastseq = seq
8195

82-
tmpbuffer[:] = _buffer[_offset+headerLen:_offset+headerLen+length]
8396

84-
try:
85-
86-
if MAGIC_COMPRESS_START==_buffer[_offset]:
87-
decompressor = zlib.decompressobj(-zlib.MAX_WBITS)
88-
tmpbuffer = decompressor.decompress(str(tmpbuffer))
97+
tmpbuffer = decompressor.decompress(str(decompress_data))
8998

90-
# _outbuffer.extend('seq:%d, hour:%d-%d len:%d decompress:%d\n' %(seq, ord(begin_hour), ord(end_hour), length, len(tmpbuffer)))
91-
except Exception, e:
92-
_outbuffer.extend("[F]decode_log_file.py decompress err, " + str(e) + "\n")
93-
return _offset+headerLen+length+1
99+
# _outbuffer.extend('seq:%d, hour:%d-%d len:%d decompress:%d\n' %(seq, ord(begin_hour), ord(end_hour), length, len(tmpbuffer)))
100+
except Exception, e:
101+
_outbuffer.extend("[F]decode_log_file.py decompress err, " + str(e) + "\n")
102+
return _offset+headerLen+length+1
94103

95-
_outbuffer.extend(tmpbuffer)
96-
97-
return _offset+headerLen+length+1
104+
_outbuffer.extend(tmpbuffer)
105+
106+
return _offset+headerLen+length+1
98107

99108

100109
def ParseFile(_file, _outfile):
101-
fp = open(_file, "rb")
102-
_buffer = bytearray(os.path.getsize(_file))
103-
fp.readinto(_buffer)
104-
fp.close()
105-
startpos = GetLogStartPos(_buffer, 2)
106-
if -1==startpos:
107-
return
108-
109-
outbuffer = bytearray()
110-
111-
while True:
112-
startpos = DecodeBuffer(_buffer, startpos, outbuffer)
113-
if -1==startpos: break;
114-
115-
if 0==len(outbuffer): return
116-
117-
fpout = open(_outfile, "wb")
118-
fpout.write(outbuffer)
119-
fpout.close()
120-
110+
fp = open(_file, "rb")
111+
_buffer = bytearray(os.path.getsize(_file))
112+
fp.readinto(_buffer)
113+
fp.close()
114+
startpos = GetLogStartPos(_buffer, 2)
115+
if -1==startpos:
116+
return
117+
118+
outbuffer = bytearray()
119+
120+
while True:
121+
startpos = DecodeBuffer(_buffer, startpos, outbuffer)
122+
if -1==startpos: break;
123+
124+
if 0==len(outbuffer): return
125+
126+
fpout = open(_outfile, "wb")
127+
fpout.write(outbuffer)
128+
fpout.close()
129+
121130
def main(args):
122-
global lastseq
123-
124-
if 1==len(args):
125-
if os.path.isdir(args[0]):
126-
filelist = glob.glob(args[0] + "/*.xlog")
127-
for filepath in filelist:
128-
lastseq = 0
129-
ParseFile(filepath, filepath+".log")
130-
else: ParseFile(args[0], args[0]+".log")
131-
elif 2==len(args):
132-
ParseFile(args[0], args[1])
133-
else:
134-
filelist = glob.glob("*.xlog")
135-
for filepath in filelist:
136-
lastseq = 0
137-
ParseFile(filepath, filepath+".log")
131+
global lastseq
132+
133+
if 1==len(args):
134+
if os.path.isdir(args[0]):
135+
filelist = glob.glob(args[0] + "/*.xlog")
136+
for filepath in filelist:
137+
lastseq = 0
138+
ParseFile(filepath, filepath+".log")
139+
else: ParseFile(args[0], args[0]+".log")
140+
elif 2==len(args):
141+
ParseFile(args[0], args[1])
142+
else:
143+
filelist = glob.glob("*.xlog")
144+
for filepath in filelist:
145+
lastseq = 0
146+
ParseFile(filepath, filepath+".log")
138147

139148
if __name__ == "__main__":
140149
main(sys.argv[1:])

‎mars/libraries/mars_android_sdk/jni/log_crypt.cc

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,20 @@
1+
// Tencent is pleased to support the open source community by making Mars available.
2+
// Copyright (C) 2016 THL A29 Limited, a Tencent company. All rights reserved.
3+
4+
// Licensed under the MIT License (the "License"); you may not use this file except in
5+
// compliance with the License. You may obtain a copy of the License at
6+
// http://opensource.org/licenses/MIT
7+
8+
// Unless required by applicable law or agreed to in writing, software distributed under the License is
9+
// distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
10+
// either express or implied. See the License for the specific language governing permissions and
11+
// limitations under the License.
12+
113
//
214
// log_crypt.cc
315
// mars-ext
416
//
5-
// Created by yanguoyue on 16/6/14.
17+
// Created by garry on 16/6/14.
618
// Copyright © 2016年 Tencent. All rights reserved.
719
//
820

@@ -14,7 +26,7 @@
1426
#include <time.h>
1527

1628
static const char kMagicSyncStart = '\x03';
17-
static const char kMagicAsyncStart ='\x04';
29+
static const char kMagicAsyncStart ='\x05';
1830
static const char kMagicEnd = '\0';
1931

2032
static uint16_t __GetSeq(bool _is_async) {
@@ -299,7 +311,6 @@ void LogCrypt::CryptSyncLog(const char* const _log_data, size_t _input_len, char
299311
void LogCrypt::CryptAsyncLog(const char* const _log_data, size_t _input_len, char* _output, size_t& _output_len) {
300312
_output_len = std::min(_input_len, _output_len);
301313
memcpy(_output, _log_data, _output_len);
302-
303314
}
304315

305316
bool LogCrypt::Fix(char* _data, size_t _data_len, bool& _is_async, uint32_t& _raw_log_len) {

‎mars/libraries/mars_android_sdk/jni/log_crypt.cc.rewriteme

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
// log_crypt.cc
1515
// mars-ext
1616
//
17-
// Created by yanguoyue on 16/6/14.
17+
// Created by garry on 16/6/14.
1818
// Copyright © 2016年 Tencent. All rights reserved.
1919
//
2020

@@ -24,10 +24,9 @@
2424
#include <string.h>
2525
#include <sys/time.h>
2626
#include <time.h>
27-
#include <stdio.h>
2827

2928
static const char kMagicSyncStart = '\x03';
30-
static const char kMagicAsyncStart ='\x04';
29+
static const char kMagicAsyncStart ='\x05';
3130
static const char kMagicEnd = '\0';
3231

3332
static uint16_t __GetSeq(bool _is_async) {
@@ -312,7 +311,6 @@ void LogCrypt::CryptSyncLog(const char* const _log_data, size_t _input_len, char
312311
void LogCrypt::CryptAsyncLog(const char* const _log_data, size_t _input_len, char* _output, size_t& _output_len) {
313312
_output_len = std::min(_input_len, _output_len);
314313
memcpy(_output, _log_data, _output_len);
315-
316314
}
317315

318316
bool LogCrypt::Fix(char* _data, size_t _data_len, bool& _is_async, uint32_t& _raw_log_len) {

‎mars/libraries/mars_xlog_sdk/jni/log_crypt.cc

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,20 @@
1+
// Tencent is pleased to support the open source community by making Mars available.
2+
// Copyright (C) 2016 THL A29 Limited, a Tencent company. All rights reserved.
3+
4+
// Licensed under the MIT License (the "License"); you may not use this file except in
5+
// compliance with the License. You may obtain a copy of the License at
6+
// http://opensource.org/licenses/MIT
7+
8+
// Unless required by applicable law or agreed to in writing, software distributed under the License is
9+
// distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
10+
// either express or implied. See the License for the specific language governing permissions and
11+
// limitations under the License.
12+
113
//
214
// log_crypt.cc
315
// mars-ext
416
//
5-
// Created by yanguoyue on 16/6/14.
17+
// Created by garry on 16/6/14.
618
// Copyright © 2016年 Tencent. All rights reserved.
719
//
820

@@ -14,7 +26,7 @@
1426
#include <time.h>
1527

1628
static const char kMagicSyncStart = '\x03';
17-
static const char kMagicAsyncStart ='\x04';
29+
static const char kMagicAsyncStart ='\x05';
1830
static const char kMagicEnd = '\0';
1931

2032
static uint16_t __GetSeq(bool _is_async) {
@@ -299,7 +311,6 @@ void LogCrypt::CryptSyncLog(const char* const _log_data, size_t _input_len, char
299311
void LogCrypt::CryptAsyncLog(const char* const _log_data, size_t _input_len, char* _output, size_t& _output_len) {
300312
_output_len = std::min(_input_len, _output_len);
301313
memcpy(_output, _log_data, _output_len);
302-
303314
}
304315

305316
bool LogCrypt::Fix(char* _data, size_t _data_len, bool& _is_async, uint32_t& _raw_log_len) {

0 commit comments

Comments
 (0)