hack.lu CTF 2021 Writeups
Table of Contents
Tenbagger #
Challenge description:
I think I took it too far and made some trades and lost everything. My only chance to fix my account balance is a tenbagger.
We are given a pcap and open in Wireshark. It contains a lot of what looks like normal web browsing. But somewhere in there are a few FIX messages. FIX is the Financial Information eXchange protocol. First, I thought that we need to get the credentials from the FIX login, but there are no such packages. It is much simpler than that. In the text field, spread over multiple packages, there is the flag in plain text:
flag{t0_th3_m00n_4nd_b4ck}
Touchy Logger #
Challenge description:
Manfred always has the best insides for stock investments, but he hates to share it with his beloved brother. Can you help me out?
We are given a libinput log file. As we would expect from the challenge title, it consists of touch events – both swiping and tapping.
A teammate (who knows how html canvases work) and I (who has no idea about fancy web stuff) solved the challenge together.
Visualizing the input on an HMTL canvas, gives us the following image:
The dots at the bottom are interesting, because they are looking like typing on a screen keyboard. Drawing boxes around it looks like:
Because of personal familiarity with it, we guessed that this must be the Ubuntu Gnome onscreen keyboard. And as we have the temporal order in the log, we can get the keys. Actually, there was a lot of tweaking the boxes and the annoyance of changing the keyboard layers involved until we got the correct text:
yfirefox
reddit.com r stocks
Twitter.com
r#StocksToBuy
hevolution
wolfgang47@stocks.iveThanks!!1!!11!!&Hi Wolfgang,Hi Wolfgang,
Count me in??? sounds like a great investment (:
Returnr rate of + -50% souns ds liketo good to be true.
I will invest 1337.69$
Hope Arnold did not mess with my Laptop again to steal my shit (-.-')
BTW using a Tablet now. This so called "keylogger" cannot work if there is no keyboard ;)
Best Regards,
Manfredhttps: investment24.flu.xxx
ffluxmanfredOiVyi)=wi$?;Ezq-lZx#
Because of the random password, the detection must be good. Visiting the website with the credentials, gives us the flag. Here is the visualization code:
script.js
(function () {
const draw = document.getElementById('draw')
const ctx = draw.getContext('2d')
const log = document.getElementById('input').value
const parsed = document.getElementById('parsed')
parsed.value = ''
let location = { x: 0, y: 0 }
const width = 13
const hideCtrl = true
const keyBoxes = []
const mkKey = (k, x, y, w = null, h = null) => {
w = w || width
h = h || width;
[x, y, w, h] = [x, y, w, h].map(e => e * 2)
keyBoxes.push(
{ sx: x, sy: y, ex: x + w, ey: y + h, k }
)
}
const findKey = (px, py) => {
const key = keyBoxes.find(({ sx, sy, ex, ey }) => {
return sx < px && sy < py && ex > px && ey > py
})
return key ? key.k : null
}
const logKey = (px, py) => {
let key = findKey(px, py)
if (key === null) {
return
}
if (lMap[mode].hasOwnProperty(key)) {
key = lMap[mode][key]
}
if (!key.includes('<') && hideCtrl) {
parsed.value += shift ? key.toUpperCase() : key
shift = false
return
}
shift = key === '<shiftl>' || key === '<shiftr>';
if (key === '<l2>') {
mode = 'l2'
}
if (key === '<l3>') {
mode = 'l3'
}
if (key === '<l1>') {
mode = 'l1'
}
}
const l1 = ',qwertyuiopasdfghjklzxcvbnm'.split('')
const l2 = `_1234567890@#$%&-+()*"':;!?`.split('')
const l3 = ``
let shift = false
const l2Map = Object.fromEntries(l1.map((k, i) => [k, l2[i]]))
//const l3Map = Object.fromEntries(l2.map((k, i) => [k, l3[i]]))
const l3Map = {}
l2Map['<l2>'] = '<l1>'
l2Map['<shiftl>'] = '<l3>'
l2Map['<shiftr>'] = '<l3>'
l3Map['<l2>'] = '<l1>'
l3Map['<shiftl>'] = '<l2>'
l3Map['<shiftr>'] = '<l2>'
l3Map['j'] = '='
let mode = 'l1'
const lMap = {
l1: {},
l2: l2Map,
l3: l3Map
}
console.log(lMap)
'qwertyuiop'.split('').forEach((k, i) => {
mkKey(k, 53 + i * width, 120)
})
mkKey('<bak>', 183, 120, 25)
'asdfghjkl'.split('').forEach((k, i) => {
mkKey(k, 60 + width * i, 133)
})
mkKey('\n', 177, 133, 30)
mkKey('<shiftl>', 45, 146, 25)
'zxcvbnm'.split('').forEach((k, i) => {
mkKey(k, 70 + i * width, 146)
})
mkKey('<shiftr>', 161, 146, 40)
mkKey('<l2>', 45, 159, 25)
mkKey(',', 70, 159)
mkKey(' ', 83, 159, 65)
mkKey('.', 148, 159, 25)
const match = l => l.match(/([0-9.]+)\/ ?([0-9.]+)mm/)
let startLocation
log.split('\n').map(l => l.trim()).forEach(l => {
if (l.includes('TOUCH_DOWN')) {
const cord = match(l)
if (cord === null) {
console.log(l)
return
}
location = { x: cord[1], y: cord[2] }
startLocation = { x: cord[1], y: cord[2] }
} else if (l.includes('TOUCH_MOTION')) {
const cord = match(l)
if (cord === null) {
console.log(l)
return
}
location = { x: cord[1], y: cord[2] }
} else if (l.includes('TOUCH_UP')) {
const endKey = findKey(location.x * 2, location.y * 2)
const startKey = findKey(startLocation.x * 2, startLocation.y * 2)
if (startKey && endKey && startKey === endKey) {
ctx.beginPath()
ctx.moveTo(startLocation.x * 2, startLocation.y * 2)
ctx.lineTo(location.x * 2, location.y * 2)
ctx.stroke()
}
logKey(location.x * 2, location.y * 2)
//location = { x: 0, y: 0 }
}
})
console.log(keyBoxes)
ctx.font = '10px serif';
ctx.strokeStyle = 'red'
keyBoxes.forEach(({ sx, sy, ex, ey, k }) => {
ctx.fillText(k, sx + 5, sy + 8)
ctx.strokeRect(sx, sy, ex - sx, ey - sy)
})
})()
visual.html
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Title</title>
</head>
<body>
<textarea id="parsed" style="width: 800px; height: 600px"></textarea>
<br>
<canvas width="520" height="350" id="draw"></canvas>
<textarea id="input" style="display: none">
[...long log omitted...]
</textarea>
<script src="script.js"></script>
</body>
</html>
PYCOIN #
Challenge description:
A friend gave me this and he says he can not reverse this… but this is just python?
We are given a .pyc file. We can decompile it with uncompyle6 pycoin.pyc and get
import marshal
marshalled = b'\xe3\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00@\x00\x00\x00\xf3\xf0\x01\x00\x00n\x02tcd\x00d\x01l\x00m\x01Z\x01\x01\x00e\x02e\x03d\x02\x83\x01\x83\x01\xa0\x04\xa1\x00Z\x05e\x06e\x05\x83\x01d\x03k\x02\x90\x01o\xcee\x05d\x00\x19\x00d\x04k\x02\x90\x01o\xcee\x05d\x05\x19\x00e\x05d\x00\x19\x00d\x06\x17\x00k\x02\x90\x01o\xcee\x05d\x07\x19\x00e\x05d\x05\x19\x00e\x05d\x00\x19\x00\x18\x00d\x08n\n]\x02n\x02n\x02r\x04q\x04\x17\x00k\x02\x90\x01o\xcee\x05d\t\x19\x00d\nk\x02\x90\x01o\xcee\x05d\x0b\x19\x00e\x05d\x0c\x19\x00d\t\x14\x00d\r\x18\x00k\x02\x90\x01o\xcee\x05d\x0e\x19\x00e\x07e\x05\x83\x01d\x0f\x18\x00k\x02\x90\x01o\xcee\x05d\x06\x19\x00e\x05d\x10\x19\x00\x17\x00e\x05d\x11\x19\x00\x17\x00d\x12k\x02\x90\x01o\xcee\x08e\te\x05d\x10\x19\x00\x83\x01d\x07\x14\x00\x83\x01d\x05\x17\x00e\x05d\x13\x19\x00k\x02\x90\x01o\xcee\x05d\x14\x19\x00d\x15\x16\x00d\x03k\x02\x90\x01o\xcee\x05d\x13\x19\x00e\x05d\x14\x19\x00d\x07\x14\x00k\x02\x90\x01o\xcee\x01e\x05d\x11\x19\x00d\x16\x14\x00\x83\x01\xa0\n\xa1\x00d\x00\x19\x00d\x05\x18\x00e\x05d\t\x19\x00k\x02\x02\x00\x02\x00\x90\x01o\xcee\x05d\x0c\x19\x00d\x17k\x02\x90\x01o\xcee\x05d\x18\x19\x00e\x05d\x19\x19\x00d\x07\x1b\x00d\x07\x18\x00k\x02\x90\x01o\xcee\x05d\x1a\x19\x00e\x05d\x11\x19\x00e\x05d\x14\x19\x00\x14\x00d\x1b\x16\x00d\x07\x14\x00d\x05\x18\x00k\x02\x90\x01o\xcee\x05d\x19\x19\x00e\x05d\x18\x19\x00e\x05d\x13\x19\x00A\x00e\x05d\x1c\x19\x00A\x00d\t\x14\x00d\x1d\x18\x00k\x02\x90\x01o\xcee\x05d\x1c\x19\x00d\x1ek\x02Z\x0be\x0ce\x0b\x90\x01r\xe6d\x1fe\x05\xa0\r\xa1\x00\x9b\x00\x9d\x02n\x02d \x83\x01\x01\x00d!S\x00)"\xe9\x00\x00\x00\x00)\x01\xda\x03md5z\x1aplease supply a valid key:\xe9\x10\x00\x00\x00\xe9f\x00\x00\x00\xe9\x01\x00\x00\x00\xe9\x06\x00\x00\x00\xe9\x02\x00\x00\x00\xe9[\x00\x00\x00\xe9\x03\x00\x00\x00\xe9g\x00\x00\x00\xe9\x04\x00\x00\x00\xe9\x0b\x00\x00\x00\xe9*\x00\x00\x00\xe9\x05\x00\x00\x00i*\x05\x00\x00\xe9\x07\x00\x00\x00\xe9\n\x00\x00\x00i\x04\x01\x00\x00\xe9\t\x00\x00\x00\xe9\x08\x00\x00\x00\xe9\x11\x00\x00\x00\xf3\x01\x00\x00\x00a\xe97\x00\x00\x00\xe9\x0c\x00\x00\x00\xe9\x0e\x00\x00\x00\xe9\r\x00\x00\x00\xe9 \x00\x00\x00\xe9\x0f\x00\x00\x00\xe9\x17\x00\x00\x00\xe9}\x00\x00\x00z\x0bvalid key! z\x0einvalid key :(N)\x0eZ\x07hashlibr\x03\x00\x00\x00\xda\x03str\xda\x05input\xda\x06encode\xda\x01k\xda\x03len\xda\x03sum\xda\x03int\xda\x03chr\xda\x06digestZ\x07correct\xda\x05print\xda\x06decode\xa9\x00r)\x00\x00\x00r)\x00\x00\x00\xfa\r<disassembly>\xda\x08<module>\x01\x00\x00\x00sF\x00\x00\x00\x0c\x02\x10\x03\x0e\x01\n\xff\x04\x02\x12\xfe\x04\x03\x1a\xfd\x04\x04\n\xfc\x04\x05\x16\xfb\x04\x06\x12\xfa\x04\x07\x1a\xf9\x04\x08\x1e\xf8\x04\t\x0e\xf7\x04\n\x12\xf6\x04\x0b"\xf5\x04\x0c\n\xf4\x04\r\x16\xf3\x04\x0e"\xf2\x04\x0f&\xf1\x04\x10\n\xee\x02\x15'
exec(marshal.loads(marshalled))
Great! Python bytecode in python bytecode.
Prepending the pyc magic bytes and a date field to the marshalled bytes
head = b'\x55\x0d\x0d\x0a\x00\x00\x00\x00\x66\x39\x7d\x61\x15\x00\x00\x00'
l = marshal.loads(marshalled)
with open('co_code.pyc', 'wb+') as f:
f.write(head + l.co_code)
allows us to save the program as as a working pyc file again.
Doing uncompyle6 co_code.pyc does not work. We found out the reason for this only after the CTF was over. They injected invalid instructions, so that the program would still work, but uncompyle6 and even pythons dis would not work. During the CTF, we solved it with manual reversing. After the CTF I managed to get dis to work, but this handcrafted bytecode could not be decompyled by any tool I tried.
marshalled = marshalled.replace(b'n\x02t', b'\x09\x09\x09', 1)
marshalled.replace(b'\x02\x02', b'\x09\x09', 1)
marshalled.replace(b'n\n]\x02n\x02', b'\x09' * 6, 1)
By the way, you can get opcodes from dis.opmap. E.g.,opmap['NOT'] = 9
Here is the disassembly with some of my comments
L. 1 0 JUMP_FORWARD 4 'to 4'
2 LOAD_GLOBAL 99 99
4_0 COME_FROM 114 '114'
4_1 COME_FROM 112 '112'
4_2 COME_FROM 0 '0'
4 LOAD_CONST 0
6 LOAD_CONST ('md5',)
8 IMPORT_NAME hashlib
10 IMPORT_FROM md5
L. 3 12 STORE_NAME md5
14 POP_TOP
16 LOAD_NAME str
18 LOAD_NAME input
20 LOAD_STR 'please supply a valid key:'
22 CALL_FUNCTION_1 1 ''
24 CALL_FUNCTION_1 1 ''
26 LOAD_METHOD encode
L. 6 28 CALL_METHOD_0 0 ''
30 STORE_NAME k
32 LOAD_NAME len
34 LOAD_NAME k
36 CALL_FUNCTION_1 1 ''
38 LOAD_CONST 16
40 COMPARE_OP ==
# here is the start of the 16 compares
# k[0] = 102 = ord('f')
L. 7 42_44 JUMP_IF_FALSE_OR_POP 462 'to 462'
46 LOAD_NAME k
48 LOAD_CONST 0
50 BINARY_SUBSCR
L. 6 52 LOAD_CONST 102
54 COMPARE_OP ==
# k[1] = k[0] + 6 = 108 = ord('l')
L. 8 56_58 JUMP_IF_FALSE_OR_POP 462 'to 462'
60 LOAD_NAME k
62 LOAD_CONST 1
64 BINARY_SUBSCR
66 LOAD_NAME k
68 LOAD_CONST 0
70 BINARY_SUBSCR
72 LOAD_CONST 6
L. 6 74 BINARY_ADD
76 COMPARE_OP ==
L. 9 78_80 JUMP_IF_FALSE_OR_POP 462 'to 462'
# assume k[2] = ord('a')
82 LOAD_NAME k
84 LOAD_CONST 2
86 BINARY_SUBSCR
88 LOAD_NAME k
90 LOAD_CONST 1
92 BINARY_SUBSCR
94 LOAD_NAME k
96 LOAD_CONST 0
98 BINARY_SUBSCR
100 BINARY_SUBTRACT # first - second
102 LOAD_CONST 91
L. 6 104 JUMP_FORWARD 116 'to 116'
106 FOR_ITER 110 'to 110'
L. 10 108 JUMP_FORWARD 112 'to 112'
110_0 COME_FROM 106 '106'
110 BREAK_LOOP 114 'to 114'
112_0 COME_FROM 108 '108'
112 POP_JUMP_IF_FALSE_BACK 4 'to 4'
114_0 COME_FROM 110 '110'
114 JUMP_BACK 4 'to 4'
116_0 COME_FROM 104 '104'
116 BINARY_ADD
L. 6 118 COMPARE_OP ==
120_122 JUMP_IF_FALSE_OR_POP 462 'to 462'
# k[3] = 103 = ord('g')
124 LOAD_NAME k
126 LOAD_CONST 3
128 BINARY_SUBSCR
130 LOAD_CONST 103
132 COMPARE_OP ==
134_136 JUMP_IF_FALSE_OR_POP 462 'to 462'
# assume k[4] = 123 = ord('{')
# k[4] = k[11] * k[3] - 42 => k[11] = 55
138 LOAD_NAME k
140 LOAD_CONST 4
142 BINARY_SUBSCR
L. 6 144 LOAD_NAME k
146 LOAD_CONST 11
L. 12 148 BINARY_SUBSCR
150 LOAD_CONST 3
152 BINARY_MULTIPLY
154 LOAD_CONST 42
156 BINARY_SUBTRACT
158 COMPARE_OP ==
160_162 JUMP_IF_FALSE_OR_POP 462 'to 462'
# from hint we know k[5] = 53 = ord('5')
# sum(k) - 1322 = 53
164 LOAD_NAME k
L. 6 166 LOAD_CONST 5
168 BINARY_SUBSCR
L. 13 170 LOAD_NAME sum
172 LOAD_NAME k
174 CALL_FUNCTION_1 1 ''
176 LOAD_CONST 1322
178 BINARY_SUBTRACT
180 COMPARE_OP ==
182_184 JUMP_IF_FALSE_OR_POP 462 'to 462'
# k[6] + k[7] + k[10] = 260
186 LOAD_NAME k
188 LOAD_CONST 6
190 BINARY_SUBSCR # push k[6]
192 LOAD_NAME k
194 LOAD_CONST 7
L. 6 196 BINARY_SUBSCR # push k[7]
198 BINARY_ADD # pop; pop; push k[6] + k[7]
L. 14 200 LOAD_NAME k
202 LOAD_CONST 10
204 BINARY_SUBSCR # push k[10]
206 BINARY_ADD
208 LOAD_CONST 260
210 COMPARE_OP ==
212_214 JUMP_IF_FALSE_OR_POP 462 'to 462'
# k[9] == int(chr(k[7]) * 2) + 1
# k[7] is a single digit number
# k[9] \in {12, 23, 34, 45, 56, 67, 78, 89, 100}
216 LOAD_NAME int
218 LOAD_NAME chr
220 LOAD_NAME k
222 LOAD_CONST 7
224 BINARY_SUBSCR # k[7]
226 CALL_FUNCTION_1 1 '' # chr(k[7])
228 LOAD_CONST 2
L. 6 230 BINARY_MULTIPLY # chr(k[7]) * 2
232 CALL_FUNCTION_1 1 '' # int(chr(k[7]) * 2)
L. 15 234 LOAD_CONST 1
236 BINARY_ADD # int(chr(k[7]) * 2) + 1
238 LOAD_NAME k
240 LOAD_CONST 9
242 BINARY_SUBSCR
244 COMPARE_OP ==
246_248 JUMP_IF_FALSE_OR_POP 462 'to 462'
# k[8] % 17 = 16
# k[8] = 17n + 16 for one n
# Possible solutions
# 33 !
# 50 2
# 67 C
# 84 T
# 101 e
# 118 v
250 LOAD_NAME k
L. 16 252 LOAD_CONST 8
254 BINARY_SUBSCR
256 LOAD_CONST 17
258 BINARY_MODULO
260 LOAD_CONST 16
262 COMPARE_OP ==
264_266 JUMP_IF_FALSE_OR_POP 462 'to 462'
# k[8] = k[9] * 2
268 LOAD_NAME k
L. 6 270 LOAD_CONST 9
272 BINARY_SUBSCR
L. 17 274 LOAD_NAME k
276 LOAD_CONST 8
278 BINARY_SUBSCR
280 LOAD_CONST 2
282 BINARY_MULTIPLY
284 COMPARE_OP ==
286_288 JUMP_IF_FALSE_OR_POP 462 'to 462'
# k[10] = 101 = ord('e')
# 103 = k[3] = md5(k[10] * b'a').digest()[0] - 1
# import hashlib
# for i in string.printable[:-7]:
# if hashlib.md5(ord(i) * b'a').digest()[0] -1 == 103:
# print(i)
290 LOAD_NAME md5
292 LOAD_NAME k
294 LOAD_CONST 10
296 BINARY_SUBSCR
298 LOAD_CONST b'a'
300 BINARY_MULTIPLY
302 CALL_FUNCTION_1 1 ''
304 LOAD_METHOD digest
306 CALL_METHOD_0 0 ''
L. 6 308 LOAD_CONST 0
310 BINARY_SUBSCR
L. 18 312 LOAD_CONST 1
314 BINARY_SUBTRACT
316 LOAD_NAME k
318 LOAD_CONST 3
320 BINARY_SUBSCR
L. 6 322 COMPARE_OP ==
324 ROT_TWO
L. 19 326 ROT_TWO # does nothing, swaps stack back and forth
328_330 JUMP_IF_FALSE_OR_POP 462 'to 462'
# k[11] = 55
332 LOAD_NAME k
334 LOAD_CONST 11
336 BINARY_SUBSCR
338 LOAD_CONST 55
340 COMPARE_OP ==
342_344 JUMP_IF_FALSE_OR_POP 462 'to 462'
# k[14] // 2 - 2 = k[12]
346 LOAD_NAME k
L. 6 348 LOAD_CONST 12
350 BINARY_SUBSCR
L. 20 352 LOAD_NAME k
354 LOAD_CONST 14
356 BINARY_SUBSCR
358 LOAD_CONST 2
360 BINARY_TRUE_DIVIDE
362 LOAD_CONST 2
364 BINARY_SUBTRACT
366 COMPARE_OP ==
368_370 JUMP_IF_FALSE_OR_POP 462 'to 462'
# k[13] ((k[10] * k[8] % 32) * 2) - 1
372 LOAD_NAME k
374 LOAD_CONST 13
376 BINARY_SUBSCR
378 LOAD_NAME k
380 LOAD_CONST 10
382 BINARY_SUBSCR
384 LOAD_NAME k
L. 6 386 LOAD_CONST 8
388 BINARY_SUBSCR
L. 21 390 BINARY_MULTIPLY
392 LOAD_CONST 32
394 BINARY_MODULO
396 LOAD_CONST 2
398 BINARY_MULTIPLY
400 LOAD_CONST 1
402 BINARY_SUBTRACT
404 COMPARE_OP ==
406_408 JUMP_IF_FALSE_OR_POP 462 'to 462'
# k[14] = (k[12] XOR k[9] XOR k[15]) * 3 - 23
410 LOAD_NAME k
412 LOAD_CONST 14
414 BINARY_SUBSCR
416 LOAD_NAME k
418 LOAD_CONST 12
420 BINARY_SUBSCR # k[12]
422 LOAD_NAME k
424 LOAD_CONST 9
426 BINARY_SUBSCR # k[9]
L. 6 428 BINARY_XOR # k[12] XOR k[9]
430 LOAD_NAME k
L. 22 432 LOAD_CONST 15
434 BINARY_SUBSCR
436 BINARY_XOR # k[12] XOR k[9] XOR k[15]
438 LOAD_CONST 3
440 BINARY_MULTIPLY
L. 4 442 LOAD_CONST 23
L. 25 444 BINARY_SUBTRACT
446 COMPARE_OP ==
448_450 JUMP_IF_FALSE_OR_POP 462 'to 462'
k[15] = 125 = ord('}')
452 LOAD_NAME k
454 LOAD_CONST 15
456 BINARY_SUBSCR
458 LOAD_CONST 125
460 COMPARE_OP ==
462_0 COME_FROM 448 '448'
462_1 COME_FROM 406 '406'
462_2 COME_FROM 368 '368'
462_3 COME_FROM 342 '342'
462_4 COME_FROM 328 '328'
462_5 COME_FROM 286 '286'
462_6 COME_FROM 264 '264'
462_7 COME_FROM 246 '246'
462_8 COME_FROM 212 '212'
462_9 COME_FROM 182 '182'
462_10 COME_FROM 160 '160'
462_11 COME_FROM 134 '134'
462_12 COME_FROM 120 '120'
462_13 COME_FROM 78 '78'
462_14 COME_FROM 56 '56'
462_15 COME_FROM 42 '42'
462 STORE_NAME correct
464 LOAD_NAME print
466 LOAD_NAME correct
468_470 POP_JUMP_IF_FALSE 486 'to 486'
472 LOAD_STR 'valid key! '
474 LOAD_NAME k
476 LOAD_METHOD decode
478 CALL_METHOD_0 0 ''
480 FORMAT_VALUE 0 ''
482 BUILD_STRING_2 2
484 JUMP_FORWARD 488 'to 488'
486_0 COME_FROM 468 '468'
486 LOAD_STR 'invalid key :('
488_0 COME_FROM 484 '484'
488 CALL_FUNCTION_1 1 ''
During the CTF we solved the equations by hand, but here a z3 solution from after the CTF. This is not well-written and next time I will definitely use lists:
from z3 import *
s = z3.Solver()
k0 = BitVec('k0', 16)
k1 = BitVec('k1', 16)
k2 = BitVec('k2', 16)
k3 = BitVec('k3', 16)
k4 = BitVec('k4', 16)
k5 = BitVec('k5', 16)
k6 = BitVec('k6', 16)
k7 = BitVec('k7', 16)
k8 = BitVec('k8', 16)
k9 = BitVec('k9', 16)
k10 = BitVec('k10', 16)
k11 = BitVec('k11', 16)
k12 = BitVec('k12', 16)
k13 = BitVec('k13', 16)
k14 = BitVec('k14', 16)
k15 = BitVec('k15', 16)
s.add(k0 == 102) # f
s.add(k1 == 108) # l
s.add(k2 == 97) # a
s.add(k3 == 103) # g
s.add(k4 == 123) # {
s.add(k5 == 53) # 5 (the hint that was released)
s.add(k0 + k1 + k2 + k3 + k4 + k5 + k6 + k7 + k8 + k9 + k10 + k11 + k12 + k13 + k14 + k15 - 1322 == 53)
s.add(k6 + k7 + k10 == 260)
s.add(k7 >= 49) # single digit number
s.add(k7 <= 57)
s.add((k7 - 0x30) * 11 + 1 == k9) # k9k9
s.add(k9 == 100)
s.add(k8 == 50)
s.add(k14 / 2 -2 == k12)
s.add(k10 == 101)
s.add(k13 == ((((k10 * k8) % 32) * 2) - 1))
s.add(k14 == (k12 ^ k9 ^ k15) * 3 - 23)
s.add(k11 == 55)
s.add(k15 == 125)
s.add(k0 >= ord('!'))
s.add(k0 <= ord('~'))
s.add(k1 >= ord('!'))
s.add(k1 <= ord('~'))
s.add(k2 >= ord('!'))
s.add(k2 <= ord('~'))
s.add(k3 >= ord('!'))
s.add(k3 <= ord('~'))
s.add(k4 >= ord('!'))
s.add(k4 <= ord('~'))
s.add(k5 >= ord('!'))
s.add(k5 <= ord('~'))
s.add(k6 >= ord('!'))
s.add(k6 <= ord('~'))
s.add(k8 >= ord('!'))
s.add(k8 <= ord('~'))
s.add(k9 >= ord('!'))
s.add(k9 <= ord('~'))
s.add(k10 >= ord('!'))
s.add(k10 <= ord('~'))
s.add(k11 >= ord('!'))
s.add(k11 <= ord('~'))
s.add(k12 >= ord('!'))
s.add(k12 <= ord('~'))
s.add(k13 >= ord('!'))
s.add(k13 <= ord('~'))
s.add(k14 >= ord('!'))
s.add(k14 <= ord('~'))
s.add(k15 >= ord('!'))
s.add(k15 <= ord('~'))
while s.check() == sat:
k = [k0, k1, k2, k3, k4, k5, k6, k7, k8, k9, k10, k11, k12, k13, k14, k15]
print(''.join([chr(s.model()[i].as_long()) for i in k]))
s.add(Or(*[i!=s.model()[i].as_long() for i in k]))
Thanks to the hint, we get an unique solution:
flag{5f92de703d}