[go: up one dir, main page]

brotli 3.1.3

A brotli compressor and decompressor that with an interface avoiding the rust stdlib. This makes it suitable for embedded devices and kernels. It is designed with a pluggable allocator so that the standard lib's allocator may be employed. The default build also includes a stdlib allocator and stream interface. Disable this with --features=no-stdlib. All included code is safe.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137

import sys
import os
import random as insecure_random
import subprocess
import hashlib
import traceback

def get_goal():
    ret = insecure_random.randrange(1,16)
    if ret == 15:
        ret = insecure_random.randrange(15,64)
    return ret

def shasum(work):
    sha = hashlib.sha256()
    for file in work:
        with open(file) as ff:
            while True:
                r = ff.read(65536)
                if r:
                    sha.update(r)
                else:
                    break
    return sha.digest()
def file_size(filename):
    return os.stat(filename).st_size
uncatable_stats = 0
catable_stats = 0
def process(work, brotli, cat, prefix):
    sys.stdout.flush()
    global uncatable_stats
    global catable_stats
    work = [filename for filename in work if filename.find(".jpg") != -1 or filename.find(".avi") != -1 or filename.find(".mp4") != -1 or filename.find(".mov") != -1 or insecure_random.randrange(1,17) == 16]
    if len(work) == 0:
        return
    try:
        fullsum = shasum(work)
    except Exception:
        traceback.print_exc()
        print 'ok early exit'
        return
    quality = "-q" + str(insecure_random.randrange(2,12 if len(work) < 8 else 10))
    if insecure_random.randrange(0,16) == 0:
        quality = '-q9.5'
    append = insecure_random.randrange(0,2) == 0
    frivolous_procs = []
    procs = []
    print 'processing',work,'at',quality,append
    for index, filename in enumerate(work):
        thread_count = insecure_random.randrange(1,17)
        if quality > 8 and thread_count == 1:
            thread_count = 2
        magic = insecure_random.randrange(0,2) == 0
        buffer_size = insecure_random.randrange(1,18)
        if buffer_size == 17:
            buffer_size = 65536
        elif buffer_size > 3:
            buffer_size *= 1024
        bs = "-bs" + str(buffer_size)
        args = [brotli, "-c", bs, quality]
        if magic:
            args.append("-magic")
        frivolous_procs.append(subprocess.Popen(args + [filename, prefix + quality+"-" + str(index)+".compressed"]))
        if append and index == 0:
            args.append("-appendable")
        else:
            args.append("-catable")
        args.append('-j' + str(thread_count))
        args.append(filename)
        args.append(prefix + quality+"-" + str(index)+".br")
        procs.append(subprocess.Popen(args))
    for index, proc in enumerate(procs):
        ret = proc.wait()
        if ret:
            print 'failure at ' + work[index],quality,append, bs
        assert not ret
    buffer_size = insecure_random.randrange(1,18)
    if buffer_size == 17:
        buffer_size = 65536
    elif buffer_size > 3:
        buffer_size *= 1024
    bs = "-bs" + str(buffer_size)
    args = [cat, bs]
    for index, filename in enumerate(work):
        args.append(prefix + quality+"-" + str(index)+".br")
    stdout, _stderr = subprocess.Popen(args, stdout=subprocess.PIPE).communicate()
    with open(prefix+".br", 'w') as f:
        f.write(stdout)
    procs[0] = subprocess.Popen([brotli, prefix +'.br', prefix])
    ret = procs[0].wait()
    if ret:
        print 'failure at ',work,quality,append
    assert not ret
    rtsum = shasum([prefix])
    if rtsum != fullsum:
        print 'failure at ',work,quality,append
    assert rtsum == fullsum
    print 'ok',rtsum.encode('hex')
    for (index,proc) in enumerate(frivolous_procs):
        ret = proc.wait()
        assert not ret
        uncatable_stats += file_size(prefix + quality+"-" + str(index)+".compressed")
        
    catable_stats += len(stdout)
    print uncatable_stats,'/',catable_stats,(10000*uncatable_stats/catable_stats)/100.0
    try:
        for index, filename in enumerate(work):
            os.remove(prefix + quality+"-" + str(index)+".br")
            os.remove(prefix + quality+"-" + str(index)+".compressed")
        os.remove(prefix + ".br")
        os.remove(prefix)
    except Exception:
        traceback.print_exc()
    
def main():
    start = sys.argv[1]
    brotli = sys.argv[2]
    catbrotli = sys.argv[3]
    goal = get_goal()
    work =[]
    prefix="/tmp/cat-" + os.urandom(16).encode('hex')
    for root, dirnames, filenames in os.walk(start):
        for filename in filenames:
            try:
                if file_size(os.path.join(root,filename)):
                    work.append(os.path.join(root,filename))
            except Exception:
                continue
            if len(work) >= goal:
                goal = get_goal()
                process(work, brotli, catbrotli, prefix)
                work = []
    if len(work):
        process(work, brotli, catbrotli,  prefix)
if __name__=="__main__":
    main()