#!/usr/bin/env python
##WingHeader v1
###############################################################################
## File : pyREtic.py
## Description: Main module of the pyREtic suite where decompilation and
## : source reconstruction logic lives
## Created_On : Tue Aug 3 20:26:04 2010
## Created_By : Rich Smith
## Modified_On: Mon Dec 6 15:13:59 2010
## Modified_By: Rich Smith
## License : GPLv3 (Docs/LICENSE.txt)
##
## (c) Copyright 2010, Rich Smith all rights reserved.
###############################################################################
import os
import os.path
import sys
import shutil
##Allows easy relative writes to the location this dir later
MODULE_LOCATION = os.path.dirname(__file__)
[docs]class pyREtic:
"""
Main pyREtic decompilation functionality
Encompasses both a way to walk a series of objects (filesystem or memory)
as well as the way in which the bytecode is obtained (via unmarshalling or importing)
"""
def __init__(self, write_source = True, display_source = False,
project_name = "default", project_root = None, decompiler = "unpyc"):
"""
Set project name and where to dump the code produced
"""
##What is the decompiler we are using
self.decompiler = decompiler
if not project_root:
self.project_root = os.path.join(MODULE_LOCATION, "Projects" )
else:
self.project_root = project_root
##These need to be set so that the fallback procedure for future name changes doesn't fail
self.project_name = project_name.strip(os.sep)
self.project_dir = os.path.join(self.project_root, self.project_name)
self.set_projectroot(self.project_root)
try:
os.stat(self.project_dir)
##Project already exists - switching to it
self.switch_project(self.project_name)
except:
##Project does not exist - create it
self.new_project(project_name)
#print "[+] Outputting decompiled files to: %s"%(self.dump_dir)
##How to use source code produced
self.write_sourcecode = write_source
self.display_sourcecode = display_source
def _quiet_makedir(self, dirname):
"""
Make a dir, supress message if already exists
"""
try:
os.makedirs(dirname)
except Exception, err:
if "Errno 17" not in str(err):
print "[-] Problem creating output directory '%s': %s"%(dirname, err)
raise
[docs] def normalise_path(self, path):
"""
Try and expand environment variables and '~' etc that are in paths
Return:
expanded path - string
"""
return os.path.expandvars(os.path.expanduser(path))
## def set_project(self, name):
## """
## Set the variable for the *name* of the project & alter the filesyetm
## location accordingly
## Calls set_projectdir
## """
## self.prev_project_name = self.get_project()
##
## self.project_name = name.strip(os.sep)
##
## ##Adjust the project dir and sourcode dump within project dir accordingly
## return self.set_projectdir()
[docs] def get_project(self):
"""
Get the current project name
"""
return self.project_name
[docs] def switch_project(self, name, set_env = True):
"""
Set variable for the location of the project directory & call init_project
to initialise the whole project filesystem structure
"""
self.prev_project_name = self.get_project()
self.project_name = name.strip(os.sep)
##Store previous location of the project dir
self.prev_project_dir = self.get_projectdir()
##project directory location
self.project_dir = os.path.join(self.project_root, self.project_name)
##sourcode dump location inside project
self.dump_dir = os.path.join(self.project_dir, "sourcecode")
## Alter the python path to reference the new project libs dir to allow
## overiding of modules (e.g. opcode.py) + remove previous project dirs
if set_env:
old_path = os.path.join(self.prev_project_dir, "libs")
if old_path in sys.path:
sys.path.remove(old_path)
sys.path = [os.path.join(self.get_projectdir(), "libs") ] + sys.path
[docs] def get_projectdir(self):
"""
Get the directory for the project
"""
return self.project_dir
[docs] def new_project(self, name, py_version = None):
"""
Create a new project, setting the version of python it is for to the version
supplied
"""
self.switch_project(name, False)
ret = self.init_project(py_version)
## Alter the python path to reference the new project libs dir to allow
## overiding of modules (e.g. opcode.py) + remove previous project dirs
if ret:
old_path = os.path.join(self.prev_project_dir, "libs")
if old_path in sys.path:
sys.path.remove(old_path)
sys.path = [os.path.join(self.get_projectdir(), "libs") ] + sys.path
return ret
[docs] def get_project_mod_dir(self):
"""
Get the directory for the project specific modules
"""
return os.path.join(self.project_dir, "libs")
[docs] def set_projectroot(self, location):
"""
Set the root filesystem location for ALL projects, create on the filesystem
if needed
Calls set_projectdir to reassign & create the projectdir/dumps dirs etc
"""
##Store previous location of the project root
self.prev_project_root = self.get_projectroot()
location = self.normalise_path(location)
##Create the root location for ALL projects
self._quiet_makedir(location)
self.project_root = location
##Adjust the project dir and sourcode dump within project dir accordingly
return self.switch_project(self.get_project())
[docs] def get_projectroot(self):
"""
Get current project root
"""
return self.project_root
[docs] def get_dump_dir(self):
"""
Get where the source code dump is
"""
return self.dump_dir
[docs] def init_project(self, py_ver):
"""
Setup the initial project structure on the filesystem
Return False on error & True on successful creation of new project dirs
"""
#TODO - initial project creation when py_ver unknown
if not py_ver:
##initial default project with no version has been set - just use 2.5.4 modules and hope....
py_ver = "default"
try:
##Create dir structure on fs
self._quiet_makedir(os.path.join(self.get_projectdir(), "sourcecode") )
self._quiet_makedir(os.path.join(self.get_projectdir(), "pybs") )
self._quiet_makedir(os.path.join(self.get_projectdir(), "libs") )
##Create __init__.py's for dirs from which imports may occur
f = open( os.path.join(self.get_projectdir(), "libs", "__init__.py"),"wb")
f.close()
f = open( os.path.join(self.get_projectdir(), "__init__.py"),"wb")
f.close()
#TODO - 2.7 support, change to just copy entire module set ?
rt_root = os.path.join(MODULE_LOCATION, "Downloaded_Runtimes","Python-%s"%(py_ver))
stdlib_files = {"default" : ["compileall.py", "copy.py", "dis.py", "inspect.py", "opcode.py",
"py_compile.py", "struct.py", "token.py", "tokenize.py",
"traceback.py", "types.py"],
"2.5" : ["compileall.py", "copy.py", "dis.py", "inspect.py", "opcode.py",
"py_compile.py", "struct.py", "token.py", "tokenize.py",
"traceback.py", "types.py"],
"2.6" : ["_abcoll.py", "abc.py", "collections.py", "compileall.py",
"dis.py", "inspect.py", "opcode.py", "py_compile.py",
"traceback.py", "types.py"]}
for k in stdlib_files.keys():
if k not in py_ver:
continue
for f in stdlib_files[k]:
try:
shutil.copyfile(os.path.join(rt_root, "Lib", f),
os.path.join(self.get_projectdir(), "libs", f))
except Exception, err:
print "[-] Problem copying one of the stdlib files for project library: %s"%(err)
break
else:
print "[-] version of Python %s not supported. Add files to project library by hand"%(py_ver)
##Copy the opcodes .py for unpyc
#TODO - seperate decompiler specific code from main into specific stubs for each
if self.decompiler.lower() == "unpyc":
try:
##Copy the unpyc opcode table into the project libs
shutil.copyfile(os.path.join(MODULE_LOCATION, "Decompilers","unpyc", "opcodes.py"),
os.path.join(self.get_projectdir(), "libs", "opcodes.py"))
except Exception, err:
print "[-] Problem copying opocdes.py for unpyc: %s"%(err)
print "[!] You must copy an appropriate opocdes.py to %s by hand"%(os.path.join(self.get_projectdir(), "libs"))
##Save version to the project as this may be ifferent than what the runtime reports
try:
f = open(os.path.join(self.get_projectdir(), "meta"), "w")
f.write("version:%s"%(py_ver))
f.close()
except Exception, err:
print "[-] Problem writing project meta data : %s"%(err)
return True
except Exception, err:
print "[-] Problem creating project directory structure: %s"%(err)
self.restore_previous_project_structure()
return False
[docs] def restore_previous_project_structure(self):
"""
If there was a problem creating the new project structure restore the previous values
"""
self.project_name = self.prev_project_name
self.project_dir = self.prev_project_dir
self.project_root = self.prev_project_root
self.dump_dir = os.path.join(self.prev_project_dir, "sourcecode")
[docs] def fs_unmarshal(self, fs_root, depth=None):
"""
Walk the filesystem from start directory indicated & to a depth inidicated
The unmarshal technique will be used on each .pyc/.pyo found
"""
#TODO depth, make decompiler independent
from Decompilers.unpyc import liveUnPYC as live
tag = "fs_um"
self._quiet_makedir(os.path.join(self.dump_dir, tag))
lupc = live.liveUnPYC(self)
fs_root = self.normalise_path(fs_root)
##Single file supplied no need to walk the directory
if os.path.isfile(fs_root):
print "[+] Decompiling single file: %s"%(fs_root)
sc = lupc.fs_decompile(fs_root)
##Do the output style specified
if self.write_sourcecode:
filename = os.path.split(fs_root)[-1]
self._write_source(sc, "singlefile", filename, tag)
if self.display_sourcecode:
self._display_source(sc)
else:
print "[+] Decompiling directory starting at: %s"%(fs_root)
for (path, dirs, pyx) in self._fs_walk(fs_root, depth, tag):
print "[+] Decompiling via unmarshal '%s'...."%(pyx)
sc = lupc.fs_decompile(os.path.join(path,pyx))
##Do the output style specified
if self.write_sourcecode:
self._write_source(sc, path, pyx, tag)
if self.display_sourcecode:
self._display_source(sc)
[docs] def fs_objwalk(self, fs_root, depth=None):
"""
Walk the filesystem from start directory indicated & to the depth indicated
For each object and traverse into children and decompile - no unmarshaling
No access to unmarshaling required
"""
#TODO depth, make decompiler independent
from Decompilers.unpyc import liveUnPYC as live
tag = "fs_obj"
self._quiet_makedir(os.path.join(self.dump_dir, tag))
lupc = live.liveUnPYC(self)
fs_root = self.normalise_path(fs_root)
##Single file supplied no need to walk the directory
if os.path.isfile(fs_root):
#TODO single file memory traversal
print "[-] single file memory traversal not supported yet"
return
for (path, dirs, pyx) in self._fs_walk(fs_root, depth, tag):
print "[+] Decompiling in-memory '%s'...."%(pyx)
subpath, package = os.path.split(path.strip("/"))
##Lazy, add path at start of path for eay import
sys.path.insert(0, subpath)
##Attempt to import the .pyc/.pyo
to_import = "%s.%s"%(package, os.path.splitext(pyx)[0])
try:
#TODO - account for relative imports ?
print "[+] Importing: from %s import %s"%(package, os.path.splitext(pyx)[0])
try:
p_obj = __import__(package, globals(), locals(), [os.path.splitext(pyx)[0]], -1)
p_obj = eval("p_obj.%s"%(os.path.splitext(pyx)[0]))
except (ValueError, ImportError):
__import__(pyx)
p_obj = sys.modules[pyx]
print "[+] Imported %s "%(p_obj)
except Exception, err:
print "[-] Error importing %s : %s"%(to_import, err)
import traceback
traceback.print_exc()
raw_input()
sys.path.remove(subpath)
continue
try:
##Pure memory decompile
lupc.set_top_level_module(p_obj.__name__)
sc = lupc.get_py(p_obj)
except Exception, err:
import traceback
print "[-] Error decompiling %s : %s"%(p_obj, err)
traceback.print_exc()
continue
##Do the output style specified
if self.write_sourcecode:
self._write_source(sc, path, pyx, tag)
if self.display_sourcecode:
self._display_source(sc)
##Keep syspath clean - remove what we added
sys.path.remove(subpath)
[docs] def mem_objwalk(self, obj):
"""
Pure in-memory decompile.
Start at the specified object and traverse into children and decompile
No file system access, marshal access or bytecode access required
"""
#TODO make decompiler independent
from Decompilers.unpyc import liveUnPYC as live
tag = "mem_obj"
self._quiet_makedir(os.path.join(self.dump_dir, tag))
## No top level module specified so EVERYTHING encountered will be
## decompiled .....may take a while
lupc = live.liveUnPYC(self)
sc = lupc.get_py(obj)
##Do the output style specified
try:
name = "%s.py"%(obj.__name__)
except:
name = "%s.py"%(repr(obj))
if self.write_sourcecode:
self._write_source(sc, "", name , tag)
if self.display_sourcecode:
self._display_source(sc)
[docs] def blind_mirror(self, orig_module, debug = False):
"""
Mirror all the attributes of the module we are masquearding as so that we
can be called as that module would be from other areas of the code and
expected functionality is maintained
IN:
orig_module - the name of the module we are masquearding as - string
OUT:
True/False - success or failure - boolean
"""
orig = __import__(orig_module)
orig.__name__ = "orig"
for atrr in dir(orig):
if attr[:2] == "__":
continue
if debug:
print "%s -> %s.%s"%(attr, attr.__file__, attr)
exec("%s = orig.%s"%(attr, attr))
def _fs_walk(self, fs_root, depth, tag):
"""
General function to walk a file system & yield up files which are pyc's
Essentially walk & selectively yield path, dirs, file everything that matches
criteria of .pyc or .pyo
for .py just shortcut and dump
"""
print "[+] Walking filesystem from %s"%(fs_root)
for (path, dirs, files) in os.walk(fs_root):
for pyx in files:
##If the file is .py just dump it's contents - go straight to output
if os.path.splitext(pyx)[1] == ".py":
f_py = open(os.path.join(path,pyx), "rb")
sc = f_py.read()
f_py.close
##Do the output as specified
if self.write_sourcecode:
self._write_source(sc, path, pyx, tag)
if self.display_sourcecode:
self._display_source(sc)
##Decompile .pyc/.pyo
elif os.path.splitext(pyx)[1] == ".pyc" or\
os.path.splitext(pyx)[1] == ".pyo" :
yield (path, dirs, pyx)
##If the file is not .py/.pyc./.pyo skip it
else:
continue
def _display_source(self, source):
"""
Print the source code to stdout
"""
print "\n\n",source,"\n\n"
def _write_source(self, sourcecode, path, filename, tag):
"""
Write the decompiled sourcecode to disk location
"""
dump_dir = os.path.join(self.dump_dir, tag, path.strip(os.sep))
#TODO - cleanup
self._quiet_makedir(dump_dir)
location = os.path.join(dump_dir, filename)
ext = os.path.splitext(location)[1]
if ext == ".pyc":
location = location.replace(".pyc",".py")
elif ext == ".pyo":
location = location.replace(".pyo", ".py")
try:
f = open(location, "wb")
f.write(sourcecode)
f.close()
print "[+] Source code written to: %s"%location
except Exception, err:
print "[-] Problem writing sourcecode to %s [%s]"%(location, err)