74 lines
3.2 KiB
Python
74 lines
3.2 KiB
Python
# python ./data/decompress.py $TORCH_HOME/ILSVRC2012-TAR/ ./data/data/ILSVRC2012 tar
|
|
# python ./data/decompress.py $TORCH_HOME/ILSVRC2012-ZIP/ ./data/data/ILSVRC2012 zip
|
|
import os, gc, sys
|
|
from pathlib import Path
|
|
import multiprocessing
|
|
|
|
|
|
def execute(cmds, idx, num):
|
|
#print ('{:03d} :: {:03d} :: {:03d}'.format(idx, num, len(cmds)))
|
|
for i, cmd in enumerate(cmds):
|
|
if i % num == idx:
|
|
print ('{:03d} :: {:03d} :: {:03d}/{:03d} : {:}'.format(idx, num, i, len(cmds), cmd))
|
|
os.system(cmd)
|
|
|
|
|
|
def command(prefix, cmd):
|
|
#print ('{:}{:}'.format(prefix, cmd))
|
|
#if execute: os.system(cmd)
|
|
xcmd = '(echo {:}; {:}; sleep 0.1s)'.format(prefix, cmd)
|
|
return xcmd
|
|
|
|
|
|
def main(source, destination, xtype):
|
|
assert source.exists(), '{:} does not exist'.format(source)
|
|
assert (source/'train' ).exists(), '{:}/train does not exist'.format(source)
|
|
if xtype == 'tar' : assert (source/'val.tar').exists(), '{:}/val does not exist'.format(source)
|
|
elif xtype == 'zip': assert (source/'val.zip').exists(), '{:}/val does not exist'.format(source)
|
|
else : raise ValueError('invalid unzip type : {:}'.format(xtype))
|
|
#assert num_process > 0, 'invalid num_process : {:}'.format(num_process)
|
|
source = source.resolve()
|
|
destination = destination.resolve()
|
|
destination.mkdir(parents=True, exist_ok=True)
|
|
os.system('rm -rf {:}'.format(destination))
|
|
destination.mkdir(parents=True, exist_ok=True)
|
|
(destination/'train').mkdir(parents=True, exist_ok=True)
|
|
|
|
subdirs = list( (source / 'train').glob('n*') )
|
|
all_commands = []
|
|
assert len(subdirs) == 1000, 'ILSVRC2012 should contain 1000 classes instead of {:}.'.format( len(subdirs) )
|
|
if xtype == 'tar' : cmd = command('', 'tar -xf {:} -C {:}'.format(source/'val.tar', destination))
|
|
elif xtype == 'zip': cmd = command('', 'unzip -qd {:} {:}'.format(destination, source/'val.zip'))
|
|
else : raise ValueError('invalid unzip type : {:}'.format(xtype))
|
|
all_commands.append( cmd )
|
|
for idx, subdir in enumerate(subdirs):
|
|
name = subdir.name
|
|
if xtype == 'tar' : cmd = command('{:03d}/{:03d}-th: '.format(idx, len(subdirs)), 'tar -xf {:} -C {:}'.format(source/'train'/'{:}'.format(name), destination / 'train'))
|
|
elif xtype == 'zip': cmd = command('{:03d}/{:03d}-th: '.format(idx, len(subdirs)), 'unzip -qd {:} {:}'.format(destination / 'train', source/'train'/'{:}'.format(name)))
|
|
else : raise ValueError('invalid unzip type : {:}'.format(xtype))
|
|
all_commands.append( cmd )
|
|
#print ('Collect all commands done : {:} lines'.format( len(all_commands) ))
|
|
|
|
for i, cmd in enumerate(all_commands):
|
|
print(cmd)
|
|
# os.system(cmd)
|
|
# print ('{:03d}/{:03d} : {:}'.format(i, len(all_commands), cmd))
|
|
# gc.collect()
|
|
|
|
"""
|
|
records = []
|
|
for i in range(num_process):
|
|
process = multiprocessing.Process(target=execute, args=(all_commands, i, num_process))
|
|
process.start()
|
|
records.append(process)
|
|
for process in records:
|
|
process.join()
|
|
"""
|
|
|
|
|
|
if __name__ == '__main__':
|
|
assert len(sys.argv) == 4, 'invalid argv : {:}'.format(sys.argv)
|
|
source, destination = Path(sys.argv[1]), Path(sys.argv[2])
|
|
#num_process = int(sys.argv[3])
|
|
main(source, destination, sys.argv[3])
|