Skip to content

Commit f341113

Browse files
author
Kazuki Suzuki Przyborowski
committed
Small update
1 parent 60187f0 commit f341113

File tree

3 files changed

+1265
-2255
lines changed

3 files changed

+1265
-2255
lines changed

‎neofile.py‎

Lines changed: 106 additions & 215 deletions
Original file line numberDiff line numberDiff line change
@@ -1,237 +1,128 @@
11
#!/usr/bin/env python
2-
# -*- coding: UTF-8 -*-
3-
4-
from __future__ import absolute_import, division, print_function, unicode_literals, generators, with_statement, nested_scopes
5-
6-
"""
7-
neofile.py — CLI for the PyNeoFile format (.neo).
8-
9-
New:
10-
- Input '-' for list/validate/extract/repack/convert reads archive bytes from stdin.
11-
- Output '-' for create/repack/convert writes archive bytes to stdout.
12-
- Extract with '-o -' streams a TAR archive to stdout (use: `... -e -i in.neo -o - > out.tar`).
13-
"""
14-
15-
import os, sys, argparse, tempfile, tarfile, io, base64
16-
import pyneofile as N
17-
18-
__project__ = N.__project__
19-
__program_name__ = N.__program_name__
20-
__project_url__ = N.__project_url__
21-
__version_info__ = N.__version_info__
22-
__version_date_info__ = N.__version_date_info__
23-
__version_date__ = N.__version_date__
24-
__version_date_plusrc__ = N.__version_date_plusrc__
25-
__version__ = N.__version__
26-
27-
def _stdout_bin():
28-
return getattr(sys.stdout, "buffer", sys.stdout)
29-
30-
def _stdin_bin():
31-
return getattr(sys.stdin, "buffer", sys.stdin)
32-
33-
def _build_formatspecs_from_args(args):
34-
if args.format is None or args.format.lower() == "auto":
35-
return None
36-
return {
37-
"format_name": args.format,
38-
"format_magic": args.format,
39-
"format_ver": (args.formatver or "001"),
40-
"format_delimiter": (args.delimiter or "\x00"),
41-
"new_style": True,
42-
}
43-
44-
def _convert_or_fail(infile, outpath, formatspecs, checksum, compression, level):
45-
try:
46-
return N.convert_foreign_to_neo(infile, outpath, formatspecs=formatspecs,
47-
checksumtypes=(checksum, checksum, checksum),
48-
compression=compression, compression_level=level)
49-
except RuntimeError as e:
50-
msg = str(e)
51-
if "rarfile" in msg.lower():
52-
sys.stderr.write("error: RAR support requires 'rarfile'. Install via: pip install rarfile\n")
53-
elif "py7zr" in msg.lower():
54-
sys.stderr.write("error: 7z support requires 'py7zr'. Install via: pip install py7zr\n")
55-
else:
56-
sys.stderr.write("convert error: %s\n" % msg)
57-
return None
58-
except Exception as e:
59-
sys.stderr.write("convert error: %s\n" % e)
60-
return None
61-
62-
def _emit_tar_stream_from_array(arr, outfp):
63-
"""Write a tar stream to outfp from the parsed archive array (no re-compress)."""
64-
tf = tarfile.open(fileobj=outfp, mode='w|') # stream mode
65-
try:
66-
for ent in arr['ffilelist']:
67-
name = ent['fname'].lstrip('./')
68-
if ent['ftype'] == 5:
69-
ti = tarfile.TarInfo(name=name.rstrip('/') + '/')
70-
ti.type = tarfile.DIRTYPE
71-
ti.mode = ent.get('fmode', 0o755) & 0o777
72-
ti.mtime = ent.get('fmtime', 0)
73-
ti.size = 0
74-
tf.addfile(ti)
75-
else:
76-
data = ent.get('fcontent') or b''
77-
bio = io.BytesIO(data)
78-
ti = tarfile.TarInfo(name=name)
79-
ti.type = tarfile.REGTYPE
80-
ti.mode = ent.get('fmode', 0o644) & 0o777
81-
ti.mtime = ent.get('fmtime', 0)
82-
ti.size = len(data)
83-
tf.addfile(ti, fileobj=bio)
84-
finally:
85-
tf.close()
2+
# -*- coding: utf-8 -*-
3+
from __future__ import absolute_import, division, print_function, unicode_literals
4+
import sys, os, io, argparse, json
5+
6+
__program_name__ = "PyNeoFile"
7+
8+
try:
9+
import pyneofile as P
10+
except Exception as e:
11+
raise SystemExit("Failed to import core module 'pyneofile': %s" % (e,))
12+
13+
def _read_input_bytes(path):
14+
if path in (None, '-', b'-'):
15+
data = sys.stdin.buffer.read()
16+
return data
17+
with io.open(path, 'rb') as fp:
18+
return fp.read()
19+
20+
def _write_output_bytes(path, data):
21+
if path in (None, '-', b'-'):
22+
sys.stdout.buffer.write(data)
23+
return
24+
d = os.path.dirname(path)
25+
if d and not os.path.isdir(d):
26+
os.makedirs(d)
27+
with io.open(path, 'wb') as fp:
28+
fp.write(data)
8629

8730
def main(argv=None):
88-
p = argparse.ArgumentParser(description="PyNeoFile (.neo) archiver", add_help=True)
89-
p.add_argument("-V","--version", action="version", version=__program_name__ + " " + __version__)
90-
91-
p.add_argument("-i","--input", nargs="+", required=True, help="Input files/dirs or archive file ('-' = stdin for archive bytes or newline-separated paths with -c)")
92-
p.add_argument("-o","--output", default=None, help="Output file or directory ('-'=stdout for archive bytes; on -e streams a TAR)")
93-
94-
p.add_argument("-c","--create", action="store_true", help="Create a .neo archive from inputs")
95-
p.add_argument("-e","--extract", action="store_true", help="Extract an archive to --output (or stream TAR to stdout with -o -)")
96-
p.add_argument("-r","--repack", action="store_true", help="Repack an archive (change compression)")
97-
p.add_argument("-l","--list", action="store_true", help="List entries")
98-
p.add_argument("-v","--validate", action="store_true", help="Validate checksums")
99-
p.add_argument("-t","--convert", action="store_true", help="Convert zip/tar/rar/7z → .neo first")
100-
101-
p.add_argument("-F","--format", default="auto", help="Format magic (default 'auto' via pyneofile.ini)")
102-
p.add_argument("-D","--delimiter", default=None, help="Delimiter (when not using 'auto')")
103-
p.add_argument("-m","--formatver", default=None, help="Version digits (e.g. 001)")
104-
105-
p.add_argument("-P","--compression", default="auto", help="Compression: none|zlib|gzip|bz2|xz|auto")
106-
p.add_argument("-L","--level", default=None, help="Compression level/preset")
107-
p.add_argument("-C","--checksum", default="crc32", help="Checksum algorithm")
108-
p.add_argument("-s","--skipchecksum", action="store_true", help="Skip checks while reading")
109-
p.add_argument("-d","--verbose", action="store_true", help="Verbose listing")
110-
p.add_argument("-T","--text", action="store_true", help="Treat -i - as newline-separated path list when used with -c/--create")
31+
p = argparse.ArgumentParser(prog=__program_name__, description="PyNeoFile CLI (core-only)")
32+
g = p.add_mutually_exclusive_group(required=True)
33+
g.add_argument('-l', '--list', action='store_true', help='List archive entries')
34+
g.add_argument('-e', '--extract', action='store_true', help='Extract files')
35+
g.add_argument('-c', '--create', action='store_true', help='Create archive from path or stdin')
36+
g.add_argument('-r', '--repack', action='store_true', help='Repack archive, optionally changing compression')
37+
g.add_argument('--validate', action='store_true', help='Validate checksums/structure')
38+
g.add_argument('-t', '--convert', action='store_true', help='Convert foreign (zip/tar) -> neo')
39+
40+
p.add_argument('-i', '--input', required=False, help='Input path (use - for stdin)')
41+
p.add_argument('-o', '--output', required=False, help='Output path (use - for stdout)')
42+
p.add_argument('-P', '--compression', default='auto', help='Compression: auto|none|zlib|gzip|bz2|lzma')
43+
p.add_argument('-L', '--level', default=None, type=int, help='Compression level')
44+
p.add_argument('--skipchecksum', action='store_true', help='Skip content checksum verification')
45+
p.add_argument('-d', '--verbose', action='store_true', help='Verbose listing')
46+
p.add_argument('--no-json', action='store_true', help='Skip reading per-file JSON blocks (faster)')
11147

11248
args = p.parse_args(argv)
11349

114-
formatspecs = _build_formatspecs_from_args(args)
115-
inputs = args.input
116-
infile0 = inputs[0]
117-
compression = args.compression
118-
level = None if args.level in (None, "",) else int(args.level)
119-
checksum = args.checksum
120-
121-
# Determine active action
122-
actions = ["create","extract","repack","list","validate"]
123-
active = next((a for a in actions if getattr(args, a)), None)
124-
if not active:
125-
p.error("one of --create/--extract/--repack/--list/--validate is required")
126-
127-
# Helper: read archive bytes from stdin for non-create ops
128-
def _maybe_archive_bytes():
129-
if infile0 == '-':
130-
return _stdin_bin().read()
131-
return None
132-
133-
if args.create:
134-
if infile0 == '-' and not args.text:
135-
# read newline-separated paths from stdin
136-
items = [line.strip() for line in sys.stdin if line.strip() and not line.startswith('#')]
50+
if args.list:
51+
src = args.input
52+
if src in (None, '-', b'-'):
53+
data = _read_input_bytes(src)
54+
entries = P.archivefilelistfiles_neo(data, advanced=args.verbose, include_dirs=True, skipjson=True if args.no_json else True)
13755
else:
138-
items = inputs
139-
140-
if args.convert:
141-
if not args.output:
142-
p.error("--output is required (use '-' to stream to stdout)")
143-
data = _convert_or_fail(infile0, (None if args.output == '-' else args.output),
144-
formatspecs, checksum, compression, level)
145-
if data is None:
146-
return 1
147-
if args.output == '-':
148-
_stdout_bin().write(data)
149-
return 0
150-
151-
if not args.output:
152-
p.error("--output is required for --create (use '-' to stream)")
153-
out_bytes = (args.output == '-')
154-
if out_bytes:
155-
data = N.pack_neo(items, None, formatspecs=formatspecs,
156-
checksumtypes=(checksum, checksum, checksum),
157-
compression=compression, compression_level=level)
158-
_stdout_bin().write(data)
56+
entries = P.archivefilelistfiles_neo(src, advanced=args.verbose, include_dirs=True, skipjson=args.no_json)
57+
if args.verbose:
58+
for e in entries:
59+
if isinstance(e, dict):
60+
print("{type} {compression} {size} {name}".format(**e))
61+
else:
62+
print(e)
15963
else:
160-
N.pack_neo(items, args.output, formatspecs=formatspecs,
161-
checksumtypes=(checksum, checksum, checksum),
162-
compression=compression, compression_level=level)
163-
if args.verbose: sys.stderr.write("created: %s\n" % args.output)
64+
for e in entries:
65+
print(e['name'] if isinstance(e, dict) else e)
16466
return 0
16567

166-
if args.repack:
167-
src = _maybe_archive_bytes() or infile0
168-
if args.convert:
169-
if not args.output:
170-
p.error("--output is required (use '-' to stream)")
171-
data = _convert_or_fail(src, (None if args.output == '-' else args.output),
172-
formatspecs, checksum, compression, level)
173-
if data is None:
174-
return 1
175-
if args.output == '-':
176-
_stdout_bin().write(data)
177-
return 0
178-
179-
if not args.output:
180-
p.error("--output is required for --repack (use '-' to stream)")
181-
if args.output == '-':
182-
data = N.repack_neo(src, None, formatspecs=formatspecs,
183-
checksumtypes=(checksum, checksum, checksum),
184-
compression=compression, compression_level=level)
185-
_stdout_bin().write(data)
68+
if args.validate:
69+
src = args.input
70+
if src in (None, '-', b'-'):
71+
data = _read_input_bytes(src)
72+
ok, details = P.archivefilevalidate_neo(data, verbose=args.verbose, return_details=True, skipjson=args.no_json)
18673
else:
187-
N.repack_neo(src, args.output, formatspecs=formatspecs,
188-
checksumtypes=(checksum, checksum, checksum),
189-
compression=compression, compression_level=level)
190-
if args.verbose: sys.stderr.write("repacked: %s -> %s\n" % (('<stdin>' if infile0 == '-' else infile0), args.output))
191-
return 0
74+
ok, details = P.archivefilevalidate_neo(src, verbose=args.verbose, return_details=True, skipjson=args.no_json)
75+
print("OK" if ok else "BAD")
76+
if args.verbose:
77+
for d in details:
78+
print("{index} {name} {header_ok} {json_ok} {content_ok}".format(**d))
79+
return 0 if ok else 2
19280

19381
if args.extract:
194-
src = _maybe_archive_bytes() or infile0
195-
if args.output in (None, '.') and infile0 == '-':
196-
# default would attempt to mkdir '.'; fine
197-
pass
198-
if args.output == '-':
199-
# stream TAR to stdout
200-
arr = N.archive_to_array_neo(src, formatspecs=formatspecs, listonly=False,
201-
skipchecksum=args.skipchecksum, uncompress=True)
202-
_emit_tar_stream_from_array(arr, _stdout_bin())
203-
return 0
204-
outdir = args.output or "."
205-
N.unpack_neo(src, outdir, formatspecs=formatspecs, skipchecksum=args.skipchecksum, uncompress=True)
206-
if args.verbose: sys.stderr.write("extracted → %s\n" % outdir)
207-
return 0
82+
src = args.input
83+
outdir = args.output or '.'
84+
if src in (None, '-', b'-'):
85+
data = _read_input_bytes(src)
86+
ok = P.unpack_neo(data, outdir, skipchecksum=args.skipchecksum, uncompress=True)
87+
else:
88+
ok = P.unpack_neo(src, outdir, skipchecksum=args.skipchecksum, uncompress=True)
89+
return 0 if ok else 1
20890

209-
if args.list:
210-
src = _maybe_archive_bytes() or infile0
211-
names = N.archivefilelistfiles_neo(src, formatspecs=formatspecs, advanced=args.verbose, include_dirs=True)
212-
if not args.verbose:
213-
for n in names: sys.stdout.write(n + "\n")
91+
if args.create:
92+
dst = args.output or '-'
93+
if args.input in (None, '-', b'-'):
94+
data = _read_input_bytes(args.input)
95+
payload = {"stdin.bin": data}
96+
blob = P.pack_neo(payload, outfile=None, checksumtypes=('crc32','crc32','crc32'),
97+
encoding='UTF-8', compression=args.compression, compression_level=args.level)
98+
_write_output_bytes(dst, blob)
21499
else:
215-
for ent in names:
216-
sys.stdout.write("%s\t%s\t%s\t%s\n" % (
217-
ent['type'], ent['compression'], ent['size'], ent['name']
218-
))
100+
res = P.pack_neo(args.input, outfile=dst, checksumtypes=('crc32','crc32','crc32'),
101+
encoding='UTF-8', compression=args.compression, compression_level=args.level)
102+
if isinstance(res, (bytes, bytearray)):
103+
_write_output_bytes(dst, res)
219104
return 0
220105

221-
if args.validate:
222-
src = _maybe_archive_bytes() or infile0
223-
ok, details = N.archivefilevalidate_neo(src, formatspecs=formatspecs, verbose=args.verbose, return_details=True)
224-
if not args.verbose:
225-
sys.stdout.write("valid: %s\n" % ("yes" if ok else "no"))
226-
else:
227-
sys.stdout.write("valid: %s (entries: %d)\n" % ("yes" if ok else "no", len(details)))
228-
for d in details:
229-
sys.stdout.write("%4d %s h:%s j:%s c:%s\n" % (
230-
d['index'], d['name'], d['header_ok'], d['json_ok'], d['content_ok']
231-
))
106+
if args.repack:
107+
src = args.input
108+
dst = args.output or '-'
109+
res = P.repack_neo(src if src not in (None, '-', b'-') else _read_input_bytes(src),
110+
outfile=dst, checksumtypes=('crc32','crc32','crc32'),
111+
compression=args.compression, compression_level=args.level)
112+
if isinstance(res, (bytes, bytearray)):
113+
_write_output_bytes(dst, res)
232114
return 0
233115

234-
p.error("one of --create/--extract/--repack/--list/--validate is required")
116+
if args.convert:
117+
src = args.input
118+
dst = args.output or '-'
119+
if src in (None, '-', b'-'):
120+
raise SystemExit("convert requires a path input (zip/tar). Use -i <file>")
121+
res = P.convert_foreign_to_neo(src, outfile=dst, checksumtypes=('crc32','crc32','crc32'),
122+
compression=args.compression, compression_level=args.level)
123+
if isinstance(res, (bytes, bytearray)):
124+
_write_output_bytes(dst, res)
125+
return 0
235126

236127
if __name__ == "__main__":
237128
sys.exit(main())

0 commit comments

Comments
 (0)