dependencies
| (this space intentionally left almost blank) | |||||||||||||||||||||||||||||||||||||||
namespaces
| ||||||||||||||||||||||||||||||||||||||||
(ns woa.apk.aapt.parse ;; internal libs ;; common libs (:require [clojure.string :as str] [clojure.set :as set] [clojure.walk :as walk] [clojure.zip :as zip] [clojure.java.io :as io] [clojure.pprint :refer [pprint print-table]] [clojure.stacktrace :refer [print-stack-trace]]) ;; special libs (:require [clojure.java.shell :as shell :refer [sh]])) | ||||||||||||||||||||||||||||||||||||||||
(def manifest "AndroidManifest.xml") | ||||||||||||||||||||||||||||||||||||||||
porcelain | (declare get-badging get-manifest get-layout-callbacks) (declare decompile-xml get-manifest-xml) | |||||||||||||||||||||||||||||||||||||||
plumbing | (declare parse-aapt-xmltree get-nodes-from-parsed-xmltree) (declare aapt-dump-xmltree aapt-dump-badging aapt-dump-manifest aapt-dump-resources) | |||||||||||||||||||||||||||||||||||||||
get badging in Clojure data structure | (defn get-badging [apk] (let [result (atom {}) get-string-in-single-quotes #(if-let [[_ meat] (re-find #"^'([^']+)'$" %)] meat %)] ;; first pass (doseq [line (str/split-lines (aapt-dump-badging apk))] ;; only consider lines that have values (when-let [[_ label content] (re-find #"^([^:]+):([^:]+)$" line)] (let [label (keyword label)] (swap! result update-in [label] conj content)))) ;; second pass (doseq [k (keys @result)] (swap! result update-in [k] (fn [content] (when-let [first-item (first content)] (cond ;; strings (re-matches #"'[^']+'" first-item) (into #{} (map get-string-in-single-quotes content)) ;; map (re-matches #"(?:\s[^\s'][^\s=]+='[^']+')+" first-item) (into {} (for [[_ k v] (re-seq #"\s([^\s=]+)='([^']+)'" first-item)] [(keyword k) v])) ;; set (re-matches #"(?:\s'[^']+')+" first-item) (into #{} (for [[_ v] (re-seq #"\s'([^']+)'" first-item)] v)) ;; sequence (re-matches #"'[^']+',.+" first-item) (into #{} (map #(into [] (map (fn [[_ meat]] meat) (re-seq #"'([^']+)',?" %))) content))))))) @result)) | |||||||||||||||||||||||||||||||||||||||
get manifest in Clojure data structure reference: https://developer.android.com/guide/topics/manifest/manifest-intro.html | (defn get-manifest [apk] (let [parsed-manifest (parse-aapt-xmltree (aapt-dump-manifest apk)) result (atom {}) get-node-android-name (fn [node package] (-> node (get-in [:attrs :android:name]) str (#(if (.startsWith ^String % ".") (str package %) %)) keyword))] ;; <manifest> attrs (let [node (first (get-nodes-from-parsed-xmltree parsed-manifest [:manifest]))] (doseq [attr [:android:versionCode :android:versionName :package]] (swap! result assoc-in [attr] (get-in node [:attrs attr])))) (let [package (get-in @result [:package])] ;; <manifest> level (doseq [node [:uses-permission :permission]] (swap! result assoc-in [node] (set (map #(get-node-android-name % package) (get-nodes-from-parsed-xmltree parsed-manifest [:manifest node]))))) ;; <application> level (doseq [node [:activity :activity-alias :service :receiver]] (swap! result assoc-in [node] (into {} (map (fn [node] {(get-node-android-name node package) (into {} (map (fn [intent-filter-tag] [(keyword (str "intent-filter-" (name intent-filter-tag))) (set (map #(get-node-android-name % package) (get-nodes-from-parsed-xmltree (:content node) [:intent-filter intent-filter-tag])))]) [:action :category]))}) (get-nodes-from-parsed-xmltree parsed-manifest [:manifest :application node])))))) @result)) | |||||||||||||||||||||||||||||||||||||||
return layout id and their callbacks' class.method | (defn get-layout-callbacks [apk] (->> (for [[_ layout-id _1 layout-name] (re-seq #"(?x) spec\s+resource\s+ 0x([0-9a-fA-F]+)\s+ ([^:]+):layout/([^:]+): " (aapt-dump-resources apk))] [(Long/parseLong layout-id 16) (let [xml-res-name (str "res/layout/" layout-name ".xml") the-xml (parse-aapt-xmltree (aapt-dump-xmltree apk xml-res-name)) callbacks (atom #{})] (loop [worklist the-xml] (when (and worklist (not (empty? worklist))) (let [new-worklist (atom #{})] (doseq [{:keys [tag attrs content] :as work} worklist] (let [attr-keys (keys attrs) callback-keys (filter #(let [key-name (name %)] (re-matches #"android:on.+" key-name)) attr-keys) info (select-keys attrs (set/difference (set attr-keys) (set callback-keys)))] (doseq [callback-key callback-keys] (swap! callbacks conj (merge info {:view-type (name tag) :method (get attrs callback-key)})))) (swap! new-worklist into content)) (recur @new-worklist)))) @callbacks)]) (into {}))) | |||||||||||||||||||||||||||||||||||||||
get manifest in XML format | (defn get-manifest-xml [apk] (decompile-xml apk manifest)) | |||||||||||||||||||||||||||||||||||||||
decompile the binary xml on PATH in APK | (defn decompile-xml [apk path] (let [xmltree (parse-aapt-xmltree (aapt-dump-xmltree apk path)) xmltree-to-xml (fn xmltree-to-xml [indent nodes] (when (not-empty nodes) (doseq [node nodes] (let [tag (:tag node) attrs (:attrs node) content (:content node) indent-str (apply str (repeat indent " "))] (printf "%s<%s%s" indent-str (name tag) (if-not (empty? attrs) (str " " (str/join " " (map (fn [[k v]] (if (and k v) (format "%s=\"%s\"" (name k) v) "")) attrs))) "")) (if (not-empty content) (do (println ">") (xmltree-to-xml (+ indent 2) content) (printf "%s</%s>\n" indent-str (name tag))) (println " />"))))))] (with-out-str (println "<?xml version=\"1.0\" encoding=\"utf-8\"?>") (xmltree-to-xml 0 xmltree)))) | |||||||||||||||||||||||||||||||||||||||
parse aapt xmltree dump into Clojure data structure | (defn parse-aapt-xmltree [xmltree-dump] (let [lines (vec (map #(let [[_ spaces type raw] (re-find #"^(\s*)(\S):\s(.+)$" %)] {:indent (count spaces) :type type :raw raw}) (str/split-lines xmltree-dump))) ;; first pass build: from lines to a tree build (fn build [lines] (when-let [lines (vec lines)] (when (not (empty? lines)) (let [start-indent (:indent (first lines)) segment-indexes (vec (concat (keep-indexed #(when (<= (:indent %2) start-indent) %1) lines) [(count lines)])) segments (map #(subvec lines (get segment-indexes %) (get segment-indexes (inc %))) (range (dec (count segment-indexes))))] (->> segments (map (fn [lines] (let [line (first lines) lines (rest lines) type (:type line) raw (:raw line)] (case type ;; Namespace "N" (let [[_ n v] (re-find #"^([^=]+)=([^=]+)$" raw)] {:type :namespace :name (str "xmlns:" n) :value v :children (build lines)}) ;; Element "E" (let [[_ name line] (re-find #"^(\S+)\s+\(line=(\d+)\)$" raw)] {:type :element :name name :line line :children (build lines)}) ;; Attribute "A" (let [[_ encoded-name bare-name quoted-value encoded-value bare-value] (re-find #"(?x) ^(?: ([^=(]+)\([^)]+\)| # encoded name ([^=(]+) # bare name) = (?: \"([^\"]+)\"| # quoted value \([^)]+\)(\S+)| # encoded value ([^\"(]\S*) # bare value) " raw)] {:type :attribute :name (or bare-name encoded-name) :value (or quoted-value encoded-value bare-value)}) ;; falls through nil)))) (keep identity) vec))))) pass (build lines)] (let [;; second pass: merge namespace/attributes into elements build (fn build [node & [immediate-namespace]] (case (:type node) ;; element :element (let [[attrs elems] (split-with #(= (:type %) :attribute) (:children node))] {:tag (keyword (:name node)) :attrs (let [attrs (into {} (mapcat #(let [the-key (keyword (:name %)) the-value (:value %)] (when (and the-key the-value) [[the-key the-value]])) attrs))] (if immediate-namespace (assoc attrs (keyword (:name immediate-namespace)) (:value immediate-namespace)) attrs)) :content (set (map build elems))}) ;; namespace :namespace (build (first (:children node)) ;; pass the immediate-namespace to its children (select-keys node [:name :value])))) pass (set (map build pass))] pass))) | |||||||||||||||||||||||||||||||||||||||
get nodes from parsed xmltree | (defn get-nodes-from-parsed-xmltree [parsed-xmltree [tag & more-tags]] (->> parsed-xmltree (filter #(= (:tag %) tag)) ((fn [nodes] (if more-tags (mapcat #(get-nodes-from-parsed-xmltree (:content %) more-tags) nodes) nodes))) set)) | |||||||||||||||||||||||||||||||||||||||
aapt dump xmltree | (defn aapt-dump-xmltree [apk asset] (:out (sh "aapt" "dump" "xmltree" apk asset))) | |||||||||||||||||||||||||||||||||||||||
aapt dump badging | (defn aapt-dump-badging [apk] (:out (sh "aapt" "dump" "badging" apk))) | |||||||||||||||||||||||||||||||||||||||
aapt dump xmltree | (defn aapt-dump-manifest [apk] (aapt-dump-xmltree apk manifest)) | |||||||||||||||||||||||||||||||||||||||
aapt dump resources | (defn aapt-dump-resources [apk] (:out (sh "aapt" "dump" "resources" apk))) | |||||||||||||||||||||||||||||||||||||||
(ns woa.apk.dex.asmdex.opcodes ;; common libs (:require [clojure.string :as str] [clojure.set :as set] [clojure.walk :as walk] [clojure.zip :as zip] [clojure.java.io :as io] [clojure.pprint :refer [pprint print-table]] [clojure.stacktrace :refer [print-stack-trace]]) ;; special libs (:require [clojure.reflect :refer [reflect]]) ;; imports (:import (org.ow2.asmdex Opcodes))) | ||||||||||||||||||||||||||||||||||||||||
(declare decode-opcode encode-opcode) | ||||||||||||||||||||||||||||||||||||||||
decode or-ed opcode from 'int' to 'set of keywords'encode or-ed opcode from 'seq of keywords' to 'int'deocde exclusive opcode from 'int' to 'keyword'encode exclusive opcode from 'keyword' to 'int'do encode/decode on opcodedecode opcodeencode opcode | (let [opcode-map {:access "ACC_" :debug "DBG_" :instruction "INSN_" :type "TYPE_" :value "VALUE_" :visibility "VISIBILITY_"} opcodes (into {} (map (fn [[tag name-prefix]] [tag (into {} (->> Opcodes reflect :members (map (comp str :name)) (filter #(.startsWith % name-prefix)) (map (fn [field-name] [(keyword (let [prettify-opcode-name (fn [name] (let [[_ name] (re-find #"^[^_]+_(.+)$" name)] (-> name str/lower-case (str/replace "_" "-"))))] (prettify-opcode-name field-name))) (eval `(. Opcodes ~(symbol field-name)))]))))]) opcode-map)) opcodes-invert (into {} (map (fn [[tag opcodes]] [tag (set/map-invert opcodes)]) opcodes))] (defn- decode-ored-opcode [opcode-type code] (let [opcode-type (keyword opcode-type) opcodes (opcode-type opcodes)] (set (filter #(not= 0 (bit-and code (get opcodes % 0))) (keys opcodes))))) (defn- encode-ored-opcode [opcode-type code] (let [opcode-type (keyword opcode-type) opcodes (opcode-type opcodes)] (reduce bit-or 0 (map opcodes (set/intersection (set code) (set (keys opcodes))))))) (defn- decode-exclusive-opcode [opcode-type code] (get ((keyword opcode-type) opcodes-invert) code)) (defn- encode-exclusive-opcode [opcode-type code] (get ((keyword opcode-type) opcodes) code)) (let [impl {:encode {:ored encode-ored-opcode :exclusive encode-exclusive-opcode} :decode {:ored decode-ored-opcode :exclusive decode-exclusive-opcode}} opcode-type-map {:access :ored :debug :exclusive :instruction :exclusive :type :exclusive :value :exclusive :visibility :exclusive}] (defn- do-opcode [dowhat opcode-type code] (let [dowhat (keyword dowhat) opcode-type (keyword opcode-type)] ((get (dowhat impl) (opcode-type opcode-type-map)) opcode-type code))) (defn decode-opcode [opcode-type code] (do-opcode :decode opcode-type code)) (defn encode-opcode [opcode-type code] (do-opcode :encode opcode-type code)))) | |||||||||||||||||||||||||||||||||||||||
(ns woa.apk.dex.asmdex.parse ;; internal libs (:require [woa.apk.dex.parse :refer [the-dex]] [woa.apk.util :refer [get-apk-file-input-stream]] [woa.apk.dex.asmdex.opcodes :refer [decode-opcode encode-opcode]]) ;; common libs (:require [clojure.string :as str] [clojure.set :as set] [clojure.walk :as walk] [clojure.zip :as zip] [clojure.java.io :as io] [clojure.pprint :refer [pprint print-table]] [clojure.stacktrace :refer [print-stack-trace]]) ;; special libs ;; imports ;; http://asm.ow2.org/doc/tutorial-asmdex.html (:import (org.ow2.asmdex ApplicationReader ApplicationVisitor ClassVisitor AnnotationVisitor FieldVisitor MethodVisitor Opcodes))) | ||||||||||||||||||||||||||||||||||||||||
declaration | ||||||||||||||||||||||||||||||||||||||||
(declare parse-the-dex-in-apk) | ||||||||||||||||||||||||||||||||||||||||
implementation | ||||||||||||||||||||||||||||||||||||||||
parse the dex in apk | (defn parse-the-dex-in-apk [apk & {:keys [] :as args}] (let [api Opcodes/ASM4 app-reader (ApplicationReader. api (get-apk-file-input-stream apk the-dex)) the-structure (atom {})] (let [app-visitor (proxy [ApplicationVisitor] [api] (visitClass [access name signature super-name interfaces] (let [access (decode-opcode :access access) signature (set signature) interfaces (set interfaces)] (swap! the-structure assoc-in [name] {:access access :name name :signature signature :super-name super-name :interfaces interfaces ;; placeholders :fields {} :methods {}}) (let [class-name name] (proxy [ClassVisitor] [api] (visitField [access name desc signature value] (let [access (decode-opcode :access access) signature (set signature)] (swap! the-structure assoc-in [class-name :fields name] {:access access :name name :desc desc :signature signature :value value}) nil)) (visitMethod [access name desc signature exceptions] (let [access (decode-opcode :access access) signature (set signature) exceptions (set exceptions) ;; code to be appended here code (atom [])] (swap! the-structure assoc-in [class-name :methods name] {:access access :name name :desc desc :signature signature :exceptions exceptions ;; to be filled later at the visitEnd for MethodVisitor :code nil}) (let [method-name name conj-code (fn [insn] (swap! code conj insn))] (proxy [MethodVisitor] [api] (visitArrayLengthInsn [value-reg array-reg] (conj-code {:tag :array-length-insn :instruction :array-length :array array-reg :value value-reg})) (visitArrayOperationInsn [opcode value-reg array-reg idx-reg] (conj-code {:tag :array-op-insn :instruction (decode-opcode :instruction opcode) :array array-reg :index idx-reg :value value-reg})) (visitFieldInsn [opcode owner name desc value-reg obj-reg] (conj-code {:tag :field-insn :instruction (decode-opcode :instruction opcode) :owner owner :name name :desc desc :value value-reg :object obj-reg})) (visitFillArrayDataInsn [array-reg array-data] (conj-code {:tag :fill-array-data-insn :instruction :fill-array-data :array array-reg :data array-data})) (visitInsn [opcode] (conj-code {:tag :nullary-insn :instruction (decode-opcode :instruction opcode)})) (visitIntInsn [opcode reg] (conj-code {:tag :unary-insn :instruction (decode-opcode :instruction opcode) :reg reg})) (visitJumpInsn [opcode label reg-a reg-b] (conj-code {:tag :jump-insn :instruction (decode-opcode :instruction opcode) :label label :reg-a reg-a :reg-b reg-b})) (visitLabel [label] (conj-code {:tag :label :label label})) (visitLineNumber [line start] (conj-code {:tag :line-number :line line :start start})) (visitLocalVariable [name desc signature start end index] (conj-code {:tag :local-variable :name name :desc desc :signature signature :start start :end end :index index})) (visitLookupSwitchInsn [reg default switch-keys labels] (let [switch-keys (vec switch-keys) labels (vec labels)] (conj-code {:tag :lookup-switch-insn :instruction :lookup-switch :reg reg :default default :switch-keys switch-keys :labels labels}))) (visitMaxs [max-stack _] ;; local vars + param vars (last ones; "this" implicit for instance method) (swap! the-structure assoc-in [class-name :methods method-name :vars] max-stack)) (visitMethodInsn [opcode owner name desc arguments] (let [arguments (vec arguments)] (conj-code {:tag :method-insn :instruction (decode-opcode :instruction opcode) :owner owner :name name :desc desc :arguments arguments}))) (visitMultiANewArrayInsn [desc regs] (let [regs (vec regs)] (conj-code {:tag :multi-a-newarray-insn :instruction :multi-a-newarray :desc desc :reg regs}))) (visitOperationInsn [opcode dest-reg src-reg-1 src-reg-2 value] (conj-code {:tag :op-insn :instruction (decode-opcode :instruction opcode) :dest-reg dest-reg :src-reg-1 src-reg-1 :src-reg-2 src-reg-2 :value value})) (visitParameters [params] (let [params (vec params)] (swap! the-structure assoc-in [class-name :methods method-name :params] params))) (visitStringInsn [opcode dest-reg string] (conj-code {:tag :string-insn :instruction (decode-opcode :instruction opcode) :dest-reg dest-reg :string string})) (visitTableSwitchInsn [reg min max default labels] (let [labels (vec labels)] (conj-code {:tag :table-switch-insn :instruction :table-switch :reg reg :min min :max max :default default :labels labels}))) (visitTryCatchBlock [start end handler type] (conj-code {:tag :try-catch-block :start start :end end :handler handler :type type})) (visitTypeInsn [opcode dest-reg ref-reg size-reg type] (conj-code {:tag :type-insn :instruction (decode-opcode :instruction opcode) :ref-reg ref-reg :size-reg size-reg :type type})) (visitVarInsn [opcode dest-reg var] (conj-code {:tag :var-insn :instruction (decode-opcode :instruction opcode) :dest-reg dest-reg :var var})) (visitEnd [] ;; now save the code (swap! the-structure assoc-in [class-name :methods method-name :code] @code)))))))))))] (.accept app-reader app-visitor (bit-or 0 ApplicationReader/SKIP_DEBUG)) @the-structure))) | |||||||||||||||||||||||||||||||||||||||
(ns woa.apk.dex.asmdex.util ;; internal libs ;; common libs (:require [clojure.string :as str] [clojure.set :as set] [clojure.walk :as walk] [clojure.zip :as zip] [clojure.java.io :as io] [clojure.pprint :refer [pprint print-table]] [clojure.stacktrace :refer [print-stack-trace]]) ;; special libs ;; imports) | ||||||||||||||||||||||||||||||||||||||||
declaration | ||||||||||||||||||||||||||||||||||||||||
var | ||||||||||||||||||||||||||||||||||||||||
func | (declare get-component-callback-method-all-external-invokes expand-invokes extract-dex-method-invokes comp-name-2-class-name class-name-2-comp-name) | |||||||||||||||||||||||||||||||||||||||
implementation | ||||||||||||||||||||||||||||||||||||||||
for each component callback method (i.e., on*), get its external invokes | (defn get-component-callback-method-all-external-invokes [apk] (let [expanded-invokes (expand-invokes (extract-dex-method-invokes apk))] (into {} (map (fn [comp-type] [comp-type (into {} (map (fn [comp-name] [comp-name (->> (get expanded-invokes ;; internal class name (comp-name-2-class-name comp-name)) ;; filter event callbacks (filter #(re-find #"^on[A-Z]" (first %))) ;; filter external invokes (map (fn [[k methods]] [k (set (filter (fn [{:keys [class]}] ;; external invokes (not (get expanded-invokes class))) methods))])) (into {}))]) (->> apk :manifest comp-type keys (map name))))]) [:receiver :service :activity])))) | |||||||||||||||||||||||||||||||||||||||
expand invokes to include transitive/indirect ones | (defn expand-invokes [invokes] (let [expand (fn expand [method visited] (if-let [method-invokes (get invokes method)] ;; internal method - further expansion if not visited (if-not (contains? visited method) ;; not visited (mapcat #(expand % (conj (set visited) method)) method-invokes) ;; already visited - return method #{method}) ;; external method - no further expansion #{method}))] (let [result (atom {}) tmp (->> invokes (map (fn [[k v]] [k (set (mapcat #(expand % #{%}) v))])) (into {}))] (doseq [[{:keys [class method]} all-invokes] tmp] (swap! result assoc-in [class method] all-invokes)) @result))) | |||||||||||||||||||||||||||||||||||||||
extract invokes in each method | (defn extract-dex-method-invokes [apk] (let [dex (:dex apk)] (->> (mapcat (fn [[class-name {:keys [methods]}]] (map (fn [[method-name {:keys [code]}]] {{:class class-name :method method-name} (->> code (filter #(= (:tag %) :method-insn)) (map #(do {:class (:owner %) :method (:name %) ;;:instruction (:instruction %) ; not interesting to us })) set)}) methods)) dex) (reduce merge)))) | |||||||||||||||||||||||||||||||||||||||
a.b.c -> La/b/c; | (defn comp-name-2-class-name [comp-name] (str "L" (str/replace comp-name "." "/") ";")) | |||||||||||||||||||||||||||||||||||||||
La/b/c; -> a.b.c | (defn class-name-2-comp-name [class-name] (let [[_ class-name] (re-find #"^L([^;]+);$" class-name)] (str/replace class-name "/" "."))) | |||||||||||||||||||||||||||||||||||||||
(ns woa.apk.dex.parse (:require [woa.apk.util :refer [get-apk-file-bytes get-apk-file-input-stream extract-apk-file get-apk-file-sha256-digest get-file-sha256-digest]]) ;; common libs (:require [clojure.string :as str] [clojure.set :as set] [clojure.walk :as walk] [clojure.zip :as zip] [clojure.java.io :as io] [clojure.pprint :refer [pprint print-table]] [clojure.stacktrace :refer [print-stack-trace]])) | ||||||||||||||||||||||||||||||||||||||||
declaration | ||||||||||||||||||||||||||||||||||||||||
var | (declare the-dex) | |||||||||||||||||||||||||||||||||||||||
func | (declare extract-the-dex-in-apk get-the-dex-sha256-digest) | |||||||||||||||||||||||||||||||||||||||
implementation | ||||||||||||||||||||||||||||||||||||||||
(def the-dex "classes.dex") | ||||||||||||||||||||||||||||||||||||||||
extract the dex in apk to output-file-name | (defn extract-the-dex-in-apk [apk output-file-name] (extract-apk-file apk the-dex output-file-name)) | |||||||||||||||||||||||||||||||||||||||
get sha256 digest of the dex in apk | (defn get-the-dex-sha256-digest [apk] (get-apk-file-sha256-digest apk the-dex)) | |||||||||||||||||||||||||||||||||||||||
(ns woa.apk.dex.soot.parse ;; internal libs (:require [woa.util :refer [print-stack-trace-if-verbose]]) (:require [woa.apk.dex.soot.util :as util :refer :all]) (:require [woa.apk.dex.soot.simulator :as simulator :refer :all]) (:require [woa.apk.parse :as apk-parse]) ;; common libs (:require [clojure.string :as str] [clojure.set :as set] [clojure.walk :as walk] [clojure.zip :as zip] [clojure.java.io :as io] [clojure.pprint :refer [pprint print-table]] [clojure.stacktrace :refer [print-stack-trace]]) ;; special lib (:require [me.raynes.fs :as fs]) ;; imports (:import (java.util.concurrent Executors TimeUnit)) (:import (soot Unit SootField SootClass SootMethod SootMethodRef) (soot.jimple Stmt) (soot.options Options))) | ||||||||||||||||||||||||||||||||||||||||
declaration | ||||||||||||||||||||||||||||||||||||||||
func | ||||||||||||||||||||||||||||||||||||||||
public | (declare parse-apk get-apk-interesting-invokes) | |||||||||||||||||||||||||||||||||||||||
private | (declare prettify-args) | |||||||||||||||||||||||||||||||||||||||
implementation | ||||||||||||||||||||||||||||||||||||||||
parse apk with soot | (defn parse-apk [apk-name options] (merge (apk-parse/parse-apk apk-name) {:dex (get-apk-interesting-invokes apk-name {} options)})) | |||||||||||||||||||||||||||||||||||||||
get App components and their (transitive) interesting invokes | (defn get-apk-interesting-invokes [apk-name {:keys [exclusion-name-patterns exclusion-name-pattern-exceptions] :as params :or {exclusion-name-patterns [#"^java\." #"^javax\." #"^junit\." #"^org\.json" #"^org\.w3c\." #"^org\.xmlpull\."] exclusion-name-pattern-exceptions [#"^android\." #"^com\.android\." #"^dalvik\." #"^java\.lang\.System" #"^java\.lang\.Class" #"^java\.lang\.ClassLoader" #"^java\.lang\.reflect" #"^java\.security"]}} {:keys [soot-android-jar-path soot-show-result soot-result-exclude-app-methods soot-parallel-jobs verbose] :as options}] (when (and apk-name (fs/readable? apk-name)) (let [apk-path (.getPath (io/file apk-name)) get-android-jar-path #(let [res-name "EMPTY" ;; hack to get "woa.jar" dir [_ path] (re-find (re-pattern (str "^file:(.*)/[^/]+!/" res-name "$")) (.getPath (io/resource res-name)))] (str/join (System/getProperty "file.separator") [path "android.jar"])) android-jar-path (if soot-android-jar-path soot-android-jar-path (get-android-jar-path)) result (atom {}) ;; the current thread's Soot context g-objgetter (new-g-objgetter)] ;; unfortunately, Singleton is so deeply embedded in Soot's implementation, we have to work in critical Section altogether (with-soot ;; use the current thread's Soot context g-objgetter ;; reset at the end to release the Soot Objects built up during the analysis true ;; the real work begins from here (when (or (not verbose) (<= verbose 1)) (mute)) (try (doto soot-options (.set_src_prec (Options/src_prec_apk)) (.set_process_dir [apk-path]) (.set_force_android_jar android-jar-path) (.set_allow_phantom_refs true) (.set_no_bodies_for_excluded true) (.set_ignore_resolution_errors true) (.set_whole_program true) (.set_output_format (Options/output_format_none))) (doto soot-phase-options) ;; do it manually --- barebone (run-body-packs :scene soot-scene :pack-manager soot-pack-manager :body-packs ["jb"] :verbose verbose) (when (and verbose (> verbose 3)) (println "body pack finished")) ;; start working on the bodies (let [step1 (fn [] (let [application-classes (get-application-classes soot-scene) android-api-descendants (->> application-classes (filter (fn [class] (->> (get-interesting-transitive-super-class-and-interface class android-api?) not-empty)))) android-api-descendant-callbacks (->> android-api-descendants (remove #(.. ^SootClass % isPhantom)) (mapcat #(->> (.. ^SootClass % getMethods) (filter (fn [method] (and (.hasActiveBody method) (re-find #"^on[A-Z]" (.getName method))))))) set)] ;; descendant relations (doseq [descendant android-api-descendants] (when (and verbose (> verbose 3)) (println "android API descendants:" descendant)) (swap! result assoc-in [(.. descendant getPackageName) (.. descendant getName)] {:android-api-ancestors (->> (for [super (get-interesting-transitive-super-class-and-interface descendant android-api?)] {:class (.. super getName) :package (.. super getPackageName)}) set)})) ;; cg will only see parts reachable from these entry points (.. soot-scene (setEntryPoints (seq android-api-descendant-callbacks))) ;; return the result {:android-api-descendants android-api-descendants})) step1-result (step1) step2 (fn [{:keys [android-api-descendants] :as prev-step-result}] (let [application-classes (get-application-classes soot-scene) application-methods (get-application-methods soot-scene) interesting-method? (memoize (fn [method] (let [method-name (-> method get-soot-name) class (-> method get-soot-class) ;;super (-> class get-transitive-super-class-and-interface) ] ;; interestingness criteria (and true (if soot-result-exclude-app-methods ;; external (not (contains? application-methods method)) true) ;; not in exclusion-name-patterns (or (->> [class] (filter (fn [x] (some #(re-find % (-> x get-soot-class-name)) exclusion-name-patterns))) empty?) ;; ... unless in exclusion-name-pattern-exceptions (->> [class] (filter (fn [x] (some #(re-find % (-> x get-soot-class-name)) exclusion-name-pattern-exceptions))) not-empty)) ;; not <init> or <clinit> (not (re-find #"<[^>]+>" method-name))))))] (let [pool (Executors/newFixedThreadPool soot-parallel-jobs)] (doseq [descendant android-api-descendants] (.. pool (execute (fn [] ;; impose lifecycle order on callbacks ;; https://developer.android.com/images/activity_lifecycle.png (let [callbacks (->> (.. ^SootClass descendant getMethods) (filter (fn [method] (and (.hasActiveBody method) (re-find #"^on[A-Z]" (.getName method))))) (sort-by #(.getName %) (fn [x y] (let [order {"onCreate" 1 "onStart" 2 "onResume" 3 ;; others "onPause" 5 "onStop" 6 "onRestart" 7 "onDestroy" 8} ox (get order x 4) oy (get order y 4)] (compare ox oy)))))] (try (doseq [callback callbacks] (let [callback-class (.. callback getDeclaringClass)] (when (and verbose (> verbose 3)) (println "app component callback:" callback)) (let [{:keys [explicit-invokes implicit-invokes component-invokes invoke-paths]} (with-simulator (initialize-classes {:classes application-classes :circumscription application-methods} options) (get-all-interesting-invokes callback interesting-method? application-methods options))] (doseq [[type invokes] [[:explicit explicit-invokes] [:implicit implicit-invokes] [:component component-invokes]]] (swap! result assoc-in [(.. callback-class getPackageName) (.. callback-class getName) :callbacks (.. callback getName) type] (->> invokes (filter #(let [{:keys [method args]} %] (soot-queryable? method))) (map #(let [{:keys [method args]} % class (-> method get-soot-class)] {:method (-> method get-soot-name) :class (-> method get-soot-class-name) :package (.. class getPackageName) :args (->> args prettify-args str)})) set))) ;; add explicit link between invokes and their Android API ancestor (let [path [(.. callback-class getPackageName) (.. callback-class getName) :callbacks (.. callback getName) :descend]] (doseq [invoke (set/union explicit-invokes implicit-invokes)] (let [method (:method invoke)] (when (soot-queryable? method) (when-not (android-api? method) (let [method-name (-> method get-soot-name) method-class (-> method get-soot-class) v {:method method-name :class (-> method get-soot-class-name) :package (.. method-class getPackageName)} ;; Android API supers supers (->> method-class get-transitive-super-class-and-interface (filter android-api?))] (when-let [super (some #(try (if (.. ^soot.SootClass % (getMethodByNameUnsafe method-name)) % false) ;; Soot implementation: Ambiguious (catch RuntimeException e %)) supers)] (let [k {:method method-name :class (-> super get-soot-class-name) :package (.. super getPackageName)}] (swap! result update-in (conj path k) #(conj (set %1) %2) v))))))))) (let [path [(.. callback-class getPackageName) (.. callback-class getName) :callbacks (.. callback getName) :invoke-paths]] (swap! result assoc-in path invoke-paths))))) (catch Exception e (print-stack-trace-if-verbose e verbose)) (catch Error e ;; any error in processing; skip this sample (.. pool shutdownNow)))))))) (.. pool shutdown) (try (when-not (.. pool (awaitTermination Integer/MAX_VALUE TimeUnit/SECONDS)) (.. pool shutdownNow)) (catch InterruptedException e (.. pool shutdownNow) (.. Thread currentThread interrupt)))) ;; must be in Soot body to ensure content/arguments can be printed (when soot-show-result (pprint @result)))) step2-result (step2 step1-result)]) ;; catch Exception to prevent disrupting outer threads (catch Exception e (print-stack-trace-if-verbose e verbose)) (finally (unmute)))) @result))) | |||||||||||||||||||||||||||||||||||||||
prettify results | (defn- prettify-args [args] (try (cond (instance? woa.apk.dex.soot.sexp.ErrorSexp args) (list (prettify-args (:type args)) (prettify-args (:info args))) (instance? woa.apk.dex.soot.sexp.ExternalSexp args) (list :instance (prettify-args (:type args))) (or (instance? woa.apk.dex.soot.sexp.BinaryOperationSexp args) (instance? woa.apk.dex.soot.sexp.UnaryOperationSexp args)) (list* (:operator args) (map prettify-args (:operands args))) (instance? woa.apk.dex.soot.sexp.InvokeSexp args) (list :invoke (prettify-args (:method args)) (prettify-args (:base args)) (prettify-args (:args args))) (instance? woa.apk.dex.soot.sexp.InstanceSexp args) (list :instance (prettify-args (:instance args))) (instance? woa.apk.dex.soot.sexp.MethodSexp args) (list :method (prettify-args (:method args))) (instance? woa.apk.dex.soot.sexp.FieldSexp args) (list :field (prettify-args (:field args))) (instance? woa.apk.dex.soot.sexp.InstanceOfSexp args) (list :instance-of (prettify-args (:class args)) (prettify-args (:instance args))) (instance? woa.apk.dex.soot.sexp.NewArraySexp args) (list :new-array (prettify-args (:base-type args)) (prettify-args (:size args))) (instance? woa.apk.dex.soot.sexp.NewMultiArraySexp args) (list :new-multi-array (prettify-args (:base-type args)) (prettify-args (:sizes args))) (instance? woa.apk.dex.soot.sexp.ArrayRefSexp args) (list :array-ref (prettify-args (:base args)) (prettify-args (:index args))) (instance? woa.apk.dex.soot.sexp.ConstantSexp args) (list :constant (prettify-args (:const args))) (or (instance? soot.SootClass args)) (list :class (get-soot-name args)) (or (instance? soot.SootMethod args) (instance? soot.SootMethodRef args)) (list :method (str (get-soot-class-name args) "." (get-soot-name args))) (or (instance? soot.SootField args) (instance? soot.SootMethodRef args)) (list :field (str (get-soot-class-name args) "." (get-soot-name args))) (soot-queryable? args) (->> args get-soot-name) (and (not (instance? woa.apk.dex.soot.sexp.Sexp args)) (coll? args)) (->> args (map prettify-args) (into (empty args))) :otherwise args) (catch Exception e args))) | |||||||||||||||||||||||||||||||||||||||
define symbolic expression (sexp) | (ns woa.apk.dex.soot.sexp ;; internal libs (:use woa.apk.dex.soot.util) ;; common libs (:require [clojure.string :as str] [clojure.set :as set] [clojure.walk :as walk] [clojure.zip :as zip] [clojure.java.io :as io] [clojure.pprint :refer [pprint print-table]] [clojure.stacktrace :refer [print-stack-trace]]) (:import (clojure.lang IHashEq))) | |||||||||||||||||||||||||||||||||||||||
declaration | ||||||||||||||||||||||||||||||||||||||||
implementation | ||||||||||||||||||||||||||||||||||||||||
(defprotocol Sexp) | ||||||||||||||||||||||||||||||||||||||||
(defrecord ErrorSexp [type info] Sexp) | ||||||||||||||||||||||||||||||||||||||||
(defn make-error-sexp [type info] (ErrorSexp. type info)) | ||||||||||||||||||||||||||||||||||||||||
(defrecord ExternalSexp [type] Sexp) | ||||||||||||||||||||||||||||||||||||||||
(defn make-external-sexp [type] (ExternalSexp. type)) | ||||||||||||||||||||||||||||||||||||||||
(defrecord BinaryOperationSexp [operator operands] Sexp) | ||||||||||||||||||||||||||||||||||||||||
(defn make-binary-operator-sexp [operator operands] (BinaryOperationSexp. operator operands)) | ||||||||||||||||||||||||||||||||||||||||
(defrecord UnaryOperationSexp [operator operands] Sexp) | ||||||||||||||||||||||||||||||||||||||||
(defn make-unary-operator-sexp [operator operands] (UnaryOperationSexp. operator operands)) | ||||||||||||||||||||||||||||||||||||||||
(defrecord InvokeSexp [invoke-type method base args] Sexp SootQuery (get-soot-class [this] (case invoke-type :static-invoke (->> (:method this) get-soot-class) (try (->> (:base this) get-soot-class) (catch Exception e (->> (:method this) get-soot-class))))) (get-soot-class-name [this] (->> this get-soot-class get-soot-name)) (get-soot-name [this] (->> (:method this) get-soot-name)) (soot-resolve [this] (->> (:method this) soot-resolve))) | ||||||||||||||||||||||||||||||||||||||||
(defn make-invoke-sexp [invoke-type method base args] (InvokeSexp. invoke-type method base args)) | ||||||||||||||||||||||||||||||||||||||||
(defrecord InstanceSexp [class instance] Sexp Object SootQuery (get-soot-class [this] (->> (:class this) get-soot-class)) (get-soot-class-name [this] (->> this get-soot-class get-soot-name)) (get-soot-name [this] (->> this get-soot-class-name)) (soot-resolve [this] (->> this get-soot-class soot-resolve))) | ||||||||||||||||||||||||||||||||||||||||
(defn make-instance-sexp [class instance] (InstanceSexp. class instance)) | ||||||||||||||||||||||||||||||||||||||||
(defrecord ClassSexp [class] Sexp Object SootQuery (get-soot-class [this] (->> (:class this) get-soot-class)) (get-soot-class-name [this] (->> this get-soot-class get-soot-name)) (get-soot-name [this] (->> this get-soot-class-name)) (soot-resolve [this] (->> this get-soot-class soot-resolve))) | ||||||||||||||||||||||||||||||||||||||||
(defn make-class-sexp [class] (ClassSexp. class)) | ||||||||||||||||||||||||||||||||||||||||
(defrecord MethodSexp [instance method] Sexp SootQuery (get-soot-class [this] (->> (:instance this) get-soot-class)) (get-soot-class-name [this] (->> this get-soot-class get-soot-name)) (get-soot-name [this] (->> (:method this) get-soot-name)) (soot-resolve [this] (->> (:method this) soot-resolve))) | ||||||||||||||||||||||||||||||||||||||||
(defn make-method-sexp [instance method] (MethodSexp. instance method)) | ||||||||||||||||||||||||||||||||||||||||
(defrecord FieldSexp [instance field] Sexp SootQuery (get-soot-class [this] (->> (:instance this) get-soot-class)) (get-soot-class-name [this] (->> this get-soot-class get-soot-name)) (get-soot-name [this] (->> (:field this) get-soot-name)) (soot-resolve [this] (->> (:field this) soot-resolve))) | ||||||||||||||||||||||||||||||||||||||||
(defn make-field-sexp [instance field] (FieldSexp. instance field)) | ||||||||||||||||||||||||||||||||||||||||
(defrecord LocalSexp [local] Sexp) | ||||||||||||||||||||||||||||||||||||||||
(defn make-local-sexp [local] (LocalSexp. local)) | ||||||||||||||||||||||||||||||||||||||||
(defrecord InstanceOfSexp [class instance] Sexp) | ||||||||||||||||||||||||||||||||||||||||
(defn make-instance-of-sexp [class instance] (InstanceOfSexp. class instance)) | ||||||||||||||||||||||||||||||||||||||||
(defrecord NewArraySexp [base-type size] Sexp) | ||||||||||||||||||||||||||||||||||||||||
(defn make-new-array-sexp [base-type size] (NewArraySexp. base-type size)) | ||||||||||||||||||||||||||||||||||||||||
(defrecord NewMultiArraySexp [base-type sizes] Sexp) | ||||||||||||||||||||||||||||||||||||||||
(defn make-new-multi-array-sexp [base-type sizes] (NewMultiArraySexp. base-type sizes)) | ||||||||||||||||||||||||||||||||||||||||
(defrecord ArrayRefSexp [base index] Sexp) | ||||||||||||||||||||||||||||||||||||||||
(defn make-array-ref-sexp [base index] (ArrayRefSexp. base index)) | ||||||||||||||||||||||||||||||||||||||||
(defrecord ConstantSexp [const] Sexp) | ||||||||||||||||||||||||||||||||||||||||
(defn make-constant-sexp [const] (ConstantSexp. const)) | ||||||||||||||||||||||||||||||||||||||||
(defrecord CastSexp [value cast-type] Sexp) | ||||||||||||||||||||||||||||||||||||||||
(defn make-cast-sexp [value cast-type] (CastSexp. value cast-type)) | ||||||||||||||||||||||||||||||||||||||||
(ns woa.apk.dex.soot.simulator ;; internal libs (:use woa.util) (:use woa.apk.dex.soot.util) (:use woa.apk.dex.soot.sexp) ;; common libs (:require [clojure.string :as str] [clojure.set :as set] [clojure.walk :as walk] [clojure.zip :as zip] [clojure.java.io :as io] [clojure.pprint :refer [pprint print-table]] [clojure.stacktrace :refer [print-stack-trace]]) ;; imports (:import (soot Unit SootField SootClass SootMethod SootMethodRef Scene) (soot.jimple Stmt StmtSwitch JimpleValueSwitch))) | ||||||||||||||||||||||||||||||||||||||||
declaration | ||||||||||||||||||||||||||||||||||||||||
public | ||||||||||||||||||||||||||||||||||||||||
(declare with-simulator) (declare initialize-classes get-all-interesting-invokes) (declare ^:dynamic *init-instances* ^:dynamic *simulator-global-state*) | ||||||||||||||||||||||||||||||||||||||||
private | ||||||||||||||||||||||||||||||||||||||||
(declare simulate-method simulate-basic-block) (declare create-simulator simulator-evaluate simulator-new-instance simulator-get-field simulator-set-field simulator-get-this simulator-get-param simulator-set-local simulator-get-local simulator-add-returns simulator-get-returns simulator-clear-returns simulator-add-explicit-invokes simulator-get-explicit-invokes simulator-clear-explicit-invokes simulator-add-implicit-invokes simulator-get-implicit-invokes simulator-clear-implicit-invokes simulator-add-component-invokes simulator-get-component-invokes simulator-clear-component-invokes simulator-add-invoke-paths simulator-get-invoke-paths simulator-clear-invoke-paths) (declare filter-implicit-cf-invoke-methods implicit-cf-class? implicit-cf? implicit-cf-task? implicit-cf-component? get-transitive-implicit-cf-super-class-and-interface get-implicit-cf-root-class-names) (declare implicit-cf-marker implicit-cf-marker-task implicit-cf-marker-component) (declare safe-invokes) | ||||||||||||||||||||||||||||||||||||||||
implementation | ||||||||||||||||||||||||||||||||||||||||
value resolver protocol | (defprotocol SimulatorValueResolver (simulator-resolve-value [obj simulator])) | |||||||||||||||||||||||||||||||||||||||
the default case | (extend-type nil SimulatorValueResolver (simulator-resolve-value [object simulator] nil)) | |||||||||||||||||||||||||||||||||||||||
(extend-type Object SimulatorValueResolver (simulator-resolve-value [object simulator] object)) | ||||||||||||||||||||||||||||||||||||||||
(extend-type soot.Local SimulatorValueResolver (simulator-resolve-value [local simulator] (let [value (simulator-get-local simulator local)] (if (= value :nil) (make-local-sexp local) (simulator-resolve-value value simulator))))) | ||||||||||||||||||||||||||||||||||||||||
(extend-type soot.SootField SimulatorValueResolver (simulator-resolve-value [field simulator] (let [instance (simulator-get-this simulator) value (simulator-get-field instance field)] (if (= value :nil) (make-field-sexp instance field) (simulator-resolve-value value simulator))))) | ||||||||||||||||||||||||||||||||||||||||
(extend-type soot.SootFieldRef SimulatorValueResolver (simulator-resolve-value [field simulator] (let [instance (simulator-get-this simulator) value (simulator-get-field instance field)] (if (= value :nil) (make-field-sexp instance field) (simulator-resolve-value value simulator))))) | ||||||||||||||||||||||||||||||||||||||||
(extend-type soot.jimple.InstanceFieldRef SimulatorValueResolver (simulator-resolve-value [field simulator] (let [instance (simulator-resolve-value (.. field getBase) simulator) field (.. field getFieldRef) value (simulator-get-field instance field)] (if (= value :nil) (make-field-sexp instance field) (simulator-resolve-value value simulator))))) | ||||||||||||||||||||||||||||||||||||||||
(extend-type soot.jimple.StaticFieldRef SimulatorValueResolver (simulator-resolve-value [field simulator] (let [value (simulator-get-field nil field)] (if (= value :nil) (make-field-sexp nil field) (simulator-resolve-value value simulator))))) | ||||||||||||||||||||||||||||||||||||||||
(extend-type soot.jimple.NullConstant SimulatorValueResolver (simulator-resolve-value [_ simulator] nil)) | ||||||||||||||||||||||||||||||||||||||||
(extend-type soot.jimple.ClassConstant SimulatorValueResolver (simulator-resolve-value [const simulator] (let [value (make-class-sexp (get-soot-class const))] value))) | ||||||||||||||||||||||||||||||||||||||||
(extend-type soot.jimple.Constant SimulatorValueResolver (simulator-resolve-value [const simulator] (let [value (try (.. const value) (catch Exception e (make-constant-sexp const)))] value))) | ||||||||||||||||||||||||||||||||||||||||
simulator assignment protocol | ||||||||||||||||||||||||||||||||||||||||
simulator should be an atom to have persistent effect | (defprotocol SimulatorAssignment (simulator-assign [target value simulator])) | |||||||||||||||||||||||||||||||||||||||
(extend-type nil SimulatorAssignment (simulator-assign [local value simulator] nil)) | ||||||||||||||||||||||||||||||||||||||||
(extend-type soot.Local SimulatorAssignment (simulator-assign [local value simulator] (swap! simulator simulator-set-local local value))) | ||||||||||||||||||||||||||||||||||||||||
(extend-type soot.SootField SimulatorAssignment (simulator-assign [field value simulator] (simulator-set-field (simulator-get-this @simulator) field value))) | ||||||||||||||||||||||||||||||||||||||||
(extend-type soot.SootFieldRef SimulatorAssignment (simulator-assign [field value simulator] (simulator-set-field (simulator-get-this @simulator) field value))) | ||||||||||||||||||||||||||||||||||||||||
(extend-type soot.jimple.FieldRef SimulatorAssignment (simulator-assign [field value simulator] (simulator-assign (.. field getFieldRef) value simulator))) | ||||||||||||||||||||||||||||||||||||||||
(extend-type soot.jimple.ArrayRef SimulatorAssignment (simulator-assign [field value simulator] (let [base (.. field getBase) base-value (-> base (simulator-resolve-value @simulator)) index-value (-> (.. field getIndex) (simulator-resolve-value @simulator))] (aset base-value index-value value)))) | ||||||||||||||||||||||||||||||||||||||||
get method interesting invokes and helpers | ||||||||||||||||||||||||||||||||||||||||
initial instance of classes within circumscription simulator's global state | (def ^:dynamic *init-instances* nil) (def ^:dynamic *simulator-global-state* nil) (defmacro with-simulator [& body] ;; initialized here to avoid unintended retention across different runs `(binding [*init-instances* (atom nil) *simulator-global-state* (atom nil)] ~@body)) | |||||||||||||||||||||||||||||||||||||||
initialize class by invoking | (defn initialize-classes [{:keys [classes circumscription] :or {circumscription :all} :as initialize-params} {:keys [verbose soot-debug-show-exceptions] :as options}] (reset! *simulator-global-state* {:fields {:static {} :instance {}}}) (let [;; soot.SootMethod cannot be reliably compared for value (as in a set) circumscription (if (= circumscription :all) circumscription (try (->> circumscription (map #(.. % getSignature)) set) (catch Exception e (set circumscription))))] (doseq [^SootClass class classes] (swap! *init-instances* assoc-in [(->> class get-soot-class-name)] (simulator-new-instance class)) (doseq [^SootMethod clinit (.. (soot.EntryPoints/v) (clinitsOf class))] (try (simulate-method {:method clinit :this nil :params nil} (assoc-in options [:circumscription] circumscription)) (catch Exception e (when soot-debug-show-exceptions (print-stack-trace e)))))))) | |||||||||||||||||||||||||||||||||||||||
get both explicit and implicit interesting invokes | (defn get-all-interesting-invokes [^SootMethod root-method interesting-method? circumscription {:keys [verbose soot-debug-show-exceptions] :as options}] (let [all-explicit-invokes (atom #{}) all-implicit-invokes (atom #{}) all-component-invokes (atom #{}) all-invoke-paths (atom nil) ;; soot.SootMethod cannot be reliably compared for value (as in a set) circumscription (if (= circumscription :all) circumscription (try (->> circumscription (map #(.. % getSignature)) set) (catch Exception e (set circumscription))))] (try (let [{:keys [returns explicit-invokes implicit-invokes component-invokes invoke-paths]} ;; full simulation (simulate-method {:method root-method :this ;; use initial instance if available (let [root-method-class (->> root-method get-soot-class) instance (get-in @*init-instances* [(->> root-method get-soot-class-name)])] (if instance instance (simulator-new-instance root-method-class))) :params (->> (.. root-method getParameterTypes) (map #(make-external-sexp %))) :interesting-method? interesting-method?} (assoc-in options [:circumscription] circumscription))] ;; interesting invokes can be explicit or implicit (swap! all-explicit-invokes into explicit-invokes) (swap! all-implicit-invokes into implicit-invokes) (swap! all-component-invokes into component-invokes) (reset! all-invoke-paths invoke-paths)) (catch Exception e (when soot-debug-show-exceptions (print-stack-trace e)))) ;; return result {:explicit-invokes @all-explicit-invokes :implicit-invokes @all-implicit-invokes :component-invokes @all-component-invokes :invoke-paths @all-invoke-paths})) | |||||||||||||||||||||||||||||||||||||||
simulate method | (defn- simulate-method [{:keys [method this params interesting-method?] :or {interesting-method? (constantly true)} :as simulation-params} {:keys [circumscription soot-basic-block-simulation-budget soot-method-simulation-depth-budget soot-debug-show-exceptions] :or {circumscription :all} :as options}] (let [method (try (soot-resolve method) (catch Exception e method)) method-name (try (.. method getSignature) (catch Exception e)) default-return #{(make-invoke-sexp :invoke method this params)}] (cond (not (instance? soot.SootMethod method)) {:returns default-return :explicit-invokes #{} :implicit-invokes #{} :component-invokes #{} :invoke-paths method-name} ;; only simulate method within circumscription (and (not= circumscription :all) (not (contains? circumscription (.. method getSignature)))) (do {:returns default-return :explicit-invokes #{} :implicit-invokes #{} :component-invokes #{} :invoke-paths method-name}) (< soot-method-simulation-depth-budget 0) (do {:returns #{(make-error-sexp :no-budget {:method method :this this :params params})} :explicit-invokes #{} :implicit-invokes #{} :component-invokes #{} :invoke-paths method-name}) ;; no method body, cannot proceed (try (.. method retrieveActiveBody) false (catch Exception e true)) (do {:returns #{(make-error-sexp :no-method-body {:method method :this this :params params})} :explicit-invokes #{} :implicit-invokes #{} :component-invokes #{} :invoke-paths method-name}) :otherwise (let [all-returns (atom #{}) all-explicit-invokes (atom #{}) all-implicit-invokes (atom #{}) all-component-invokes (atom #{}) all-invoke-paths (atom (when method-name {method-name #{}})) body (.. method getActiveBody) stmt-info (let [stmts (->> (.. body getUnits snapshotIterator) iterator-seq vec) stmt-2-index (->> stmts (map-indexed #(vector %2 %1)) (into {}))] {:stmts stmts :stmt-2-index stmt-2-index}) bb-budget (atom soot-basic-block-simulation-budget)] (process-worklist ;; the initial worklist #{{:simulator (create-simulator this params) :start-stmt (first (:stmts stmt-info))}} ;; the process (fn [worklist] ;; width-first search to prevent malicious code exhausting bb-budget (->> worklist (mapcat (fn [{:keys [simulator start-stmt]}] (when (and @bb-budget (> @bb-budget 0)) (let [{:keys [simulator next-start-stmts]} (simulate-basic-block {:simulator simulator :stmt-info stmt-info :start-stmt start-stmt :method method :interesting-method? interesting-method?} options)] (swap! bb-budget dec) (swap! all-returns into (-> simulator simulator-get-returns)) (swap! all-explicit-invokes into (-> simulator simulator-get-explicit-invokes)) (swap! all-implicit-invokes into (-> simulator simulator-get-implicit-invokes)) (swap! all-component-invokes into (-> simulator simulator-get-component-invokes)) (when method-name (swap! all-invoke-paths update-in [method-name] into (-> simulator simulator-get-invoke-paths))) ;; add the following to worklist (for [start-stmt next-start-stmts] ;; control flow sensitive! {:simulator (-> simulator simulator-clear-returns simulator-clear-explicit-invokes simulator-clear-implicit-invokes simulator-clear-component-invokes simulator-clear-invoke-paths) :start-stmt start-stmt})))))))) {:returns @all-returns :explicit-invokes @all-explicit-invokes :implicit-invokes @all-implicit-invokes :component-invokes @all-component-invokes :invoke-paths (if (empty? (get-in @all-invoke-paths [method-name])) method-name @all-invoke-paths)})))) | |||||||||||||||||||||||||||||||||||||||
simulate a basic block | (defn- simulate-basic-block [{:keys [simulator stmt-info start-stmt method interesting-method?] :as simulation-params} {:keys [soot-method-simulation-depth-budget soot-simulation-conservative-branching soot-simulation-linear-scan soot-debug-show-each-statement soot-debug-show-locals-per-statement soot-debug-show-all-per-statement soot-debug-show-exceptions verbose] :as options}] (let [simulator (atom simulator) [basic-block residue] (split-with (if soot-simulation-linear-scan ;; linear scan do not split at branching (constantly true) ;; otherwise, split at first branch or return #(let [^Stmt stmt %] (and (.. stmt fallsThrough) (not (.. stmt branches))))) (subvec (:stmts stmt-info) (get (:stmt-2-index stmt-info) start-stmt)))] ;; simulate statements in the basic block (doseq [^Stmt stmt basic-block] (try (.. stmt (apply (proxy [StmtSwitch] [] (caseAssignStmt [stmt] (let [target (.. stmt getLeftOp) value (-> (.. stmt getRightOp) (simulator-evaluate {:simulator simulator :interesting-method? interesting-method?} options))] (simulator-assign target value simulator))) (caseBreakpointStmt [stmt]) (caseEnterMonitorStmt [stmt]) (caseExitMonitorStmt [stmt]) (caseGotoStmt [stmt]) (caseIdentityStmt [stmt] (let [target (.. stmt getLeftOp) value (-> (.. stmt getRightOp) (simulator-evaluate {:simulator simulator :interesting-method? interesting-method?} options))] (simulator-assign target value simulator))) (caseIfStmt [stmt]) (caseInvokeStmt [stmt] (-> (.. stmt getInvokeExpr) (simulator-evaluate {:simulator simulator :interesting-method? interesting-method?} options))) (caseLookupSwitchStmt [stmt]) (caseNopStmt [stmt]) (caseRetStmt [stmt]) (caseReturnStmt [stmt]) (caseReturnVoidStmt [stmt]) (caseTableSwitchStmt [stmt]) (caseThrowStmt [stmt]) (defaultCase [stmt])))) (catch Exception e (when soot-debug-show-exceptions (print-stack-trace e)))) (when (or soot-debug-show-each-statement soot-debug-show-locals-per-statement soot-debug-show-all-per-statement) (println stmt) (when (or soot-debug-show-locals-per-statement soot-debug-show-all-per-statement) (println "- locals -") (pprint (:locals @simulator)) (when soot-debug-show-all-per-statement (println "- globals -") (pprint @*simulator-global-state*)) (println "----------")))) (let [return (atom {:simulator nil ; to be filled at the end :next-start-stmts nil}) ;; the first stmt of residue, if existed, is a brancher stmt (first residue)] (when stmt (.. stmt (apply (proxy [StmtSwitch] [] (caseAssignStmt [stmt]) (caseBreakpointStmt [stmt]) (caseEnterMonitorStmt [stmt]) (caseExitMonitorStmt [stmt]) (caseGotoStmt [^soot.jimple.internal.JGotoStmt stmt] (swap! return update-in [:next-start-stmts] conj (.. stmt getTarget))) (caseIdentityStmt [stmt]) (caseIfStmt [^soot.jimple.internal.JIfStmt stmt] (let [condition (.. stmt getCondition) value (-> condition (simulator-evaluate {:simulator simulator :interesting-method? interesting-method?} options)) target-stmt (.. stmt getTarget) next-stmt (second residue)] (if soot-simulation-conservative-branching ;; conservative branching ;; senstive to value ;; good: exact, eliminate dead branch ;; bad: may not cover enough branches when budget depelete (if-not (extends? Sexp (class value)) (if value ;; if value is true, take target-stmt (when target-stmt (swap! return update-in [:next-start-stmts] conj target-stmt)) ;; if value is false, take next-stmt (when next-stmt (swap! return update-in [:next-start-stmts] conj next-stmt))) ;; otherwise, take both stmts (doseq [stmt [next-stmt target-stmt] :when stmt] (swap! return update-in [:next-start-stmts] conj stmt))) ;; aggresive branching ;; insensitive to value ;; good: cover as much branches as possible ;; bad: not exact, may get into dead branch (doseq [stmt [next-stmt target-stmt] :when stmt] (swap! return update-in [:next-start-stmts] conj stmt))))) (caseInvokeStmt [stmt]) (caseLookupSwitchStmt [stmt]) (caseNopStmt [stmt]) (caseRetStmt [stmt]) (caseReturnStmt [stmt] (doto simulator (swap! simulator-add-returns [(-> (.. stmt getOp) (simulator-evaluate {:simulator simulator :interesting-method? interesting-method?} options))]))) (caseReturnVoidStmt [stmt] ;; nothing to do) (caseTableSwitchStmt [stmt]) (caseThrowStmt [stmt]) (defaultCase [stmt]))))) (swap! return assoc-in [:simulator] @simulator) @return))) | |||||||||||||||||||||||||||||||||||||||
frame simulator manipulators | ||||||||||||||||||||||||||||||||||||||||
(defrecord ^:private Simulator [;; for a method frame this params locals returns ;; during simulation explicit-invokes implicit-invokes component-invokes invoke-paths]) | ||||||||||||||||||||||||||||||||||||||||
(defn- create-simulator [this params] (map->Simulator {:this this :params (vec params) :locals {} :returns #{} :explicit-invokes #{} :implicit-invokes #{} :component-invokes #{} :invoke-paths #{}})) | ||||||||||||||||||||||||||||||||||||||||
(defn- simulator-new-instance [& [class]] (let [instance (gensym (str "instance" (when-let [class-name (get-soot-class-name class)] (str "-" class-name "-"))))] (make-instance-sexp class instance))) | ||||||||||||||||||||||||||||||||||||||||
evaluate expr in simulator (simulator should be an Clojure atom to allow updates) | (defn- simulator-evaluate [expr {:keys [simulator interesting-method?] :as simulation-params} {:keys [soot-method-simulation-depth-budget soot-simulation-collection-size-budget soot-no-implicit-cf soot-dump-all-invokes soot-debug-show-implicit-cf soot-debug-show-safe-invokes soot-debug-show-exceptions layout-callbacks verbose] :as options}] (let [result (atom nil) ;; binary operation binary-operator-expr (fn [expr operator operator-name] (let [op1 (-> (.. expr getOp1) (simulator-resolve-value @simulator) int) op2 (-> (.. expr getOp2) (simulator-resolve-value @simulator) int) default-return (make-binary-operator-sexp operator-name [op1 op2])] (try (operator op1 op2) (catch Exception e (when soot-debug-show-exceptions (print-stack-trace e)) default-return)))) ;; unary operation unary-operator-expr (fn [expr operator operator-name] (let [op (-> (.. expr getOp) (simulator-resolve-value @simulator) int) default-return (make-unary-operator-sexp operator-name [op])] (try (operator op) (catch Exception e (when soot-debug-show-exceptions (print-stack-trace e)) default-return)))) ;; invoke operation invoke-expr (fn [invoke-type ^SootMethodRef method base args] (let [base-value (simulator-resolve-value base @simulator) args (->> args (map #(simulator-resolve-value % @simulator)) vec) default-return (make-invoke-sexp invoke-type method base-value args) method-class (-> method get-soot-class) class-name (-> method get-soot-class-name) method-name (-> method get-soot-name)] (try ;; only add interesting methods (when (or soot-dump-all-invokes (interesting-method? method)) (doto simulator (swap! simulator-add-explicit-invokes [{:method method :args args}]))) (let [invoke-method (fn [method this params & [implicit?]] (try ;; try resolve method (soot-resolve method) (let [{:keys [returns explicit-invokes implicit-invokes component-invokes invoke-paths]} (simulate-method {:method method :this this :params params :interesting-method? interesting-method?} (update-in options [:soot-method-simulation-depth-budget] dec))] (do (doto simulator ;; implicit is contagious (swap! (if implicit? simulator-add-implicit-invokes simulator-add-explicit-invokes) explicit-invokes) (swap! simulator-add-implicit-invokes implicit-invokes) (swap! simulator-add-component-invokes component-invokes) (swap! simulator-add-invoke-paths #{invoke-paths})) ;; if the result is unique, extract it (if (== 1 (count returns)) (first returns) returns))) (catch Exception e) (finally (when (or soot-dump-all-invokes (try (interesting-method? method) (catch Exception e))) (doto simulator (swap! (if implicit? simulator-add-implicit-invokes simulator-add-explicit-invokes) [{:method method :args params}]))))))] (cond ;; safe invokes (let [t (get safe-invokes class-name)] (or (= t :all) (contains? t method-name))) (try (when soot-debug-show-safe-invokes (println "safe invoke:" class-name base-value method-name args)) (let [result (case invoke-type :special-invoke (simulator-assign base (clojure.lang.Reflector/invokeConstructor (Class/forName class-name) (object-array args)) simulator) :static-invoke (clojure.lang.Reflector/invokeStaticMethod class-name method-name (object-array args)) ;; otherwise (clojure.lang.Reflector/invokeInstanceMethod base-value method-name (object-array args)))] (when soot-debug-show-safe-invokes (println "safe invoke result:" result)) (if (and (instance? java.util.Collection result) (> (.size result) soot-simulation-collection-size-budget)) default-return result)) (catch Exception e default-return)) ;; setContentView (#{"setContentView"} method-name) (let [layout-id (first args)] (cond (number? layout-id) (doseq [{:keys [method] :as layout-callback} (get layout-callbacks layout-id)] (when layout-callback (let [info (dissoc layout-callback :method)] (try (doseq [the-method (find-method-candidates method-class method [info])] (invoke-method the-method base-value [info])) (catch Exception e default-return))))) :otherwise default-return)) ;; special-invokes (= invoke-type :special-invoke) (try (cond ;; Runnable is the one to be run (and (transitive-ancestor? "java.lang.Thread" method-class) (first args)) (simulator-assign base (first args) simulator) :otherwise (simulator-assign base (simulator-new-instance method-class) simulator)) default-return (catch Exception e default-return)) ;; implicit cf: task (and (not soot-no-implicit-cf) (implicit-cf-task? method)) (try (let [root-class-name (->> method get-implicit-cf-root-class-names first) x [root-class-name method-name]] (when soot-debug-show-implicit-cf (println "implicit cf:" x base-value args)) (cond (#{["java.lang.Thread" "start"] ["java.lang.Runnable" "run"]} x) (do (doseq [implicit-target (find-method-candidates (get-soot-class base-value) "run" [])] (when soot-debug-show-implicit-cf (println (format "implicit cf to: %1$s.%2$s:" root-class-name method-name) method-class base-value implicit-target)) (invoke-method implicit-target base-value [] true))) (#{["java.util.concurrent.Callable" "call"]} x) (doseq [implicit-target (find-method-candidates method-class "call" [])] (when soot-debug-show-implicit-cf (println (format "implicit cf to: %1$s.%2$s:" root-class-name method-name) method-class base-value implicit-target)) (invoke-method implicit-target base-value [] true)) (#{["java.util.concurrent.Executor" "execute"] ["java.util.concurrent.ExecutorService" "execute"]} x) (let [target-obj (first args)] (doseq [implicit-target (find-method-candidates (get-soot-class target-obj) "run" [])] (when soot-debug-show-implicit-cf (println (format "implicit cf to: %1$s.%2$s:" root-class-name method-name) method-class base-value implicit-target)) (invoke-method implicit-target target-obj [] true))) (#{["java.util.concurrent.ExecutorService" "submit"]} x) (let [target-obj (first args)] (doseq [implicit-target (find-method-candidates (get-soot-class target-obj) "run" [])] (when soot-debug-show-implicit-cf (println (format "implicit cf to: %1$s.%2$s:" root-class-name method-name) method-class base-value implicit-target)) (invoke-method implicit-target target-obj [] true)) (doseq [implicit-target (find-method-candidates (get-soot-class target-obj) "call" [])] (when soot-debug-show-implicit-cf (println (format "implicit cf to: %1$s.%2$s:" root-class-name method-name) method-class base-value implicit-target)) (invoke-method implicit-target target-obj [] true))) (#{["android.os.Handler" "post"] ["android.os.Handler" "postAtFrontOfQueue"] ["android.os.Handler" "postAtTime"] ["android.os.Handler" "postDelayed"]} x) (let [target-obj (first args)] (doseq [implicit-target (find-method-candidates (get-soot-class target-obj) "run" [])] (when soot-debug-show-implicit-cf (println (format "implicit cf to: %1$s.%2$s:" root-class-name method-name) method-class base-value implicit-target)) (invoke-method implicit-target target-obj [] true))) (#{["java.lang.Class" "forName"]} x) (let [target-obj (first args)] (try (-> target-obj get-soot-class) (catch Exception e (make-class-sexp target-obj)))) (#{["java.lang.Class" "getMethod"]} x) (let [target-obj (first args)] (try ;; there could be more than one such method (let [candidates (find-method-candidates (get-soot-class base-value) (str target-obj) (count (second args)))] (if-not (empty? candidates) candidates (make-method-sexp base-value target-obj))) (catch Exception e (make-method-sexp base-value target-obj)))) (#{["java.lang.reflect.Method" "invoke"]} x) (try (let [result (atom #{})] (if-not (instance? woa.apk.dex.soot.sexp.Sexp base-value) ;; try candidates (doseq [method base-value] (let [invoke-instance (first args) invoke-args (second args)] (when (= (count invoke-args) (.. method getParameterCount)) (when soot-debug-show-implicit-cf (println (format "implicit cf to: %1$s.%2$s:" root-class-name method-name) method invoke-instance invoke-args)) (when-let [r (try (invoke-method method invoke-instance invoke-args true) (catch Exception e))] (swap! result conj r))))) ;; otherwise, MethodSexp (do (doto simulator (swap! simulator-add-implicit-invokes [{:method base-value :args (second args)}])) (try (doto simulator (swap! simulator-add-invoke-paths #{(format "%1$s.%2$s[%3$d args]" (get-soot-class-name base-value) (get-soot-name base-value) (count (second args)))})) (catch Exception e)) (swap! result conj (make-invoke-sexp :reflect base-value (first args) (vec (second args)))))) (first result)) (catch Exception e (make-invoke-sexp :reflect base-value (first args) (vec (second args))))) (#{["java.lang.Class" "getField"]} x) (let [target-obj (first args)] (try (.. (-> base-value get-soot-class) (getFieldByNameUnsafe (str target-obj))) (catch Exception e (make-field-sexp base-value target-obj)))) (and (= "java.lang.reflect.Field" root-class-name) (#{"get" "getBoolean" "getByte" "getChar" "getDouble" "getFloat" "getInt" "getLong" "getShort"} method-name)) (try (let [field base-value value (simulator-get-field nil base-value)] value) (catch Exception e (make-field-sexp (simulator-get-this @simulator) base-value))) (and (= "java.lang.reflect.Field" root-class-name) (#{"equals"}) method-name) (try (let [field base-value value (simulator-get-field nil field)] (= value (first args))) (catch Exception e (make-field-sexp (simulator-get-this @simulator) base-value))) (and (= "java.lang.reflect.Field" root-class-name) (#{"set" "setBoolean" "setByte" "setChar" "setDouble" "setFloat" "setInt" "setLong" "setShort"} method-name)) (try (let [field base-value value (first args)] (simulator-set-field nil field value) value) (catch Exception e (make-field-sexp (simulator-get-this @simulator) base-value))) :default default-return)) (catch Exception e default-return)) ;; implicit cf: component (and (not soot-no-implicit-cf) (implicit-cf-component? method)) (try (let [root-class-name (->> method get-implicit-cf-root-class-names first) x [root-class-name method-name]] (cond ;; (#{["android.content.Context" "startActivity"] ;; ["android.content.Context" "startActivities"]} ;; x) ;; (update-result :category :component ;; :type "android.app.Activity" ;; :instance (with-out-str (pr (first args)))) ;; (#{["android.content.Context" "startService"] ;; ["android.content.Context" "stopService"] ;; ["android.content.Context" "bindService"] ;; ["android.content.Context" "unbindService"]} ;; x) ;; (update-result :category :component ;; :type "android.app.Service" ;; :instance (with-out-str (pr (first args)))) ;; (#{["android.content.Context" "sendBroadcast"] ;; ["android.content.Context" "sendBrocastAsUser"] ;; ["android.content.Context" "sendOrderedBroadcast"] ;; ["android.content.Context" "sendOrderedBroadcastAsUser"] ;; ["android.content.Context" "sendStickyBroadcast"] ;; ["android.content.Context" "sendStickyBroadcastAsUser"]} ;; x) ;; (update-result :category :component ;; :type "android.content.BroadcastReceiver" ;; :instance (with-out-str (pr (first args)))) ;; (#{["android.content.Context" "registerComponentCallbacks"]} ;; x) ;; (update-result :category :component ;; :type "android.content.ComponentCallbacks" ;; :instance (with-out-str (pr (first args)))) ;; (#{["android.content.Context" "registerReceiver"]} ;; x) ;; (update-result :category :component ;; :type "android.content.BroadcastReceiver" ;; :instance (with-out-str (pr args)))))))) :default default-return)) (catch Exception e default-return)) :default (invoke-method method base-value args))) (catch Exception e default-return) (finally (try (doto simulator (swap! simulator-add-invoke-paths #{(.. method getSignature)})) (catch Exception e)))))) cast-expr (fn [expr] (let [value (-> (.. expr getOp) (simulator-resolve-value @simulator)) cast-type (.. expr getCastType) default-return (make-cast-sexp value cast-type)] (try (let [result ((cond (instance? soot.BooleanType cast-type) boolean (instance? soot.ByteType cast-type) byte (instance? soot.CharType cast-type) char (instance? soot.ShortType cast-type) short (instance? soot.IntType cast-type) int (instance? soot.LongType cast-type) long (instance? soot.FloatType cast-type) float (instance? soot.DoubleType cast-type) double :otherwise identity) value)] result) (catch Exception e default-return)))) new-array-expr (fn [expr] (let [base-type (.. expr getBaseType) size (-> (.. expr getSize) (simulator-resolve-value @simulator)) default-return (make-new-array-sexp base-type size)] (try (let [result (if (< size soot-simulation-collection-size-budget) ((cond (instance? soot.BooleanType base-type) boolean-array (instance? soot.ByteType base-type) byte-array (instance? soot.CharType base-type) char-array (instance? soot.ShortType base-type) short-array (instance? soot.IntType base-type) int-array (instance? soot.LongType base-type) long-array (instance? soot.FloatType base-type) float-array (instance? soot.DoubleType base-type) double-array :otherwise object-array) size) default-return)] result) (catch Exception e default-return)))) new-multi-array-expr (fn [expr] (let [base-type (.. expr getBaseType) sizes (->> (.. expr getSizes) (map #(simulator-resolve-value % @simulator))) size (reduce * sizes) default-return (make-new-array-sexp base-type sizes)] (try (let [result (if (< size soot-simulation-collection-size-budget) (apply make-array (cond (instance? soot.BooleanType base-type) Boolean/TYPE (instance? soot.ByteType base-type) Byte/TYPE (instance? soot.CharType base-type) Character/TYPE (instance? soot.ShortType base-type) Short/TYPE (instance? soot.IntType base-type) Integer/TYPE (instance? soot.LongType base-type) Long/TYPE (instance? soot.FloatType base-type) Float/TYPE (instance? soot.DoubleType base-type) Double/TYPE :otherwise Object) sizes) default-return)] result) (catch Exception e default-return))))] (try (.. expr (apply (proxy [JimpleValueSwitch] [] ;; case local (caseLocal [local] (reset! result (simulator-resolve-value local @simulator))) ;; ConstantSwitch (caseClassConstant [const] (reset! result (simulator-resolve-value const @simulator))) (caseDoubleConstant [const] (reset! result (simulator-resolve-value const @simulator))) (caseFloatConstant [const] (reset! result (simulator-resolve-value const @simulator))) (caseIntConstant [const] (reset! result (simulator-resolve-value const @simulator))) (caseLongConstant [const] (reset! result (simulator-resolve-value const @simulator))) (caseMethodHandle [const] (reset! result (simulator-resolve-value const @simulator))) (caseNullConstant [const] (reset! result (simulator-resolve-value const @simulator))) (caseStringConstant [const] (reset! result (simulator-resolve-value const @simulator))) ;; ExprSwitch (caseAddExpr [expr] (reset! result (binary-operator-expr expr + :add))) (caseAndExpr [expr] (reset! result (binary-operator-expr expr bit-and :and))) (caseCastExpr [expr] (reset! result (cast-expr expr))) (caseCmpExpr [expr] (reset! result (binary-operator-expr expr compare :cmp))) (caseCmpgExpr [expr] ;; JVM-specific artifacts; N/A on Dalvik (reset! result (binary-operator-expr expr compare :cmpg))) (caseCmplExpr [expr] ;; JVM-specific artifacts; N/A on Dalvik (reset! result (binary-operator-expr expr compare :cmpl))) (caseDivExpr [expr] (reset! result (binary-operator-expr expr / :div))) (caseDynamicInvokeExpr [expr] ;; JVM8 specific; N/A on Dalvik (reset! result (invoke-expr :dynamic-invoke (.. expr getBootstrapMethodRef) nil (.. expr getBootstrapArgs)))) (caseEqExpr [expr] (reset! result ;; only non-sexp can be meaningfully compared (binary-operator-expr expr (fn [op1 op2] (if (and (not (extends? Sexp (class op1))) (not (extends? Sexp (class op2)))) (== op1 op2) (make-binary-operator-sexp == [op1 op2]))) :eq))) (caseGeExpr [expr] (reset! result (binary-operator-expr expr >= :ge))) (caseGtExpr [expr] (reset! result (binary-operator-expr expr > :gt))) (caseInstanceOfExpr [expr] (reset! result (let [check-type (-> (.. expr getCheckType) (simulator-resolve-value @simulator)) op (-> (.. expr getOp) (simulator-resolve-value @simulator)) default-return (make-instance-of-sexp check-type op)] (try (let [check-type-class (-> check-type get-soot-class) check-type-name (-> check-type get-soot-class-name)] (cond (instance? woa.apk.dex.soot.sexp.InvokeSexp op) (let [method (:method op) return-type (cond (instance? soot.SootMethodRef method) (.. method returnType) (instance? soot.SootMethod method) (.. method getReturnType)) type-class (-> return-type get-soot-class)] (if (transitive-ancestor? check-type-class type-class) ;; only positive answer is certain 1 default-return)) :default default-return)) (catch Exception e (when soot-debug-show-exceptions (print-stack-trace e)) default-return))))) (caseInterfaceInvokeExpr [expr] (reset! result (invoke-expr :interface-invoke (.. expr getMethodRef) (.. expr getBase) (.. expr getArgs)))) (caseLeExpr [expr] (reset! result (binary-operator-expr expr <= :le))) (caseLengthExpr [expr] (reset! result (unary-operator-expr expr count :length))) (caseLtExpr [expr] (reset! result (binary-operator-expr expr < :lt))) (caseMulExpr [expr] (reset! result (binary-operator-expr expr * :mul))) (caseNeExpr [expr] (reset! result ;; only non-sexp can be meaningfully compared (binary-operator-expr expr (fn [op1 op2] (if (and (not (extends? Sexp (class op1))) (not (extends? Sexp (class op2)))) (not= op1 op2) (make-binary-operator-sexp not= [op1 op2]))) :ne))) (caseNegExpr [expr] (reset! result (unary-operator-expr expr - :neg))) (caseNewArrayExpr [expr] (reset! result (new-array-expr expr))) (caseNewExpr [expr] ;; will be evaluated in caseSpecialInvokeExpr where the arguments are ready) (caseNewMultiArrayExpr [expr] (reset! result (new-multi-array-expr expr))) (caseOrExpr [expr] (reset! result (binary-operator-expr expr bit-or :or))) (caseRemExpr [expr] (reset! result (binary-operator-expr expr rem :rem))) (caseShlExpr [expr] (reset! result (binary-operator-expr expr not= :shl))) (caseShrExpr [expr] (reset! result (binary-operator-expr expr bit-shift-right :shr))) (caseSpecialInvokeExpr [expr] (reset! result (invoke-expr :special-invoke (.. expr getMethodRef) (.. expr getBase) (.. expr getArgs)))) (caseStaticInvokeExpr [expr] (reset! result (invoke-expr :static-invoke (.. expr getMethodRef) nil (.. expr getArgs)))) (caseSubExpr [expr] (reset! result (binary-operator-expr expr - :sub))) (caseUshrExpr [expr] (reset! result (binary-operator-expr expr unsigned-bit-shift-right :ushr))) (caseVirtualInvokeExpr [expr] (reset! result (invoke-expr :virtual-invoke (.. expr getMethodRef) (.. expr getBase) (.. expr getArgs)))) (caseXorExpr [expr] (reset! result (binary-operator-expr expr bit-xor :xor))) ;; RefSwitch (caseArrayRef [ref] (reset! result (let [base (-> (.. ref getBase) (simulator-resolve-value @simulator)) index (-> (.. ref getIndex) (simulator-resolve-value @simulator)) default-return (make-array-ref-sexp base index)] (try (aget base index) (catch Exception e (when soot-debug-show-exceptions (print-stack-trace e)) default-return))))) (caseCaughtExceptionRef [ref] ;; irrelevant) (caseInstanceFieldRef [ref] (reset! result (simulator-resolve-value ref @simulator))) (caseParameterRef [ref] (reset! result (simulator-get-param @simulator (.. ref getIndex)))) (caseStaticFieldRef [ref] (reset! result (simulator-resolve-value ref @simulator))) (caseThisRef [ref] (reset! result (simulator-get-this @simulator))) ;; default case (defaultCase [expr])))) (catch Exception e (when soot-debug-show-exceptions (print-stack-trace e)))) @result)) | |||||||||||||||||||||||||||||||||||||||
:nil signify N/A | (defn- simulator-get-field [instance field] (let [field (-> field soot-resolve) class-name (-> field get-soot-class-name) field-name (-> field get-soot-name) field-id [class-name field-name] instance (cond (instance? woa.apk.dex.soot.sexp.InstanceSexp instance) (:instance instance) :otherwise instance)] (if (.. field isStatic) (get-in @*simulator-global-state* [:fields :static field-id] :nil) (get-in @*simulator-global-state* [:fields :instance instance field-id] :nil)))) | |||||||||||||||||||||||||||||||||||||||
(defn- simulator-set-field [instance field value] (let [field (-> field soot-resolve) class-name (-> field get-soot-class-name) field-name (-> field get-soot-name) field-id [class-name field-name] instance (cond (instance? woa.apk.dex.soot.sexp.InstanceSexp instance) (:instance instance) :otherwise instance)] (if (.. field isStatic) (swap! *simulator-global-state* assoc-in [:fields :static field-id] value) (swap! *simulator-global-state* assoc-in [:fields :instance instance field-id] value)))) | ||||||||||||||||||||||||||||||||||||||||
:nil signify N/A | (defn- simulator-get-this [simulator] (get-in simulator [:this] :nil)) | |||||||||||||||||||||||||||||||||||||||
:nil signify N/A | (defn- simulator-get-param [simulator param] (get-in simulator [:params param] :nil)) | |||||||||||||||||||||||||||||||||||||||
(defn- simulator-set-local [simulator local val] (assoc-in simulator [:locals local] val)) | ||||||||||||||||||||||||||||||||||||||||
:nil signify N/A | (defn- simulator-get-local [simulator local] (get-in simulator [:locals local] :nil)) | |||||||||||||||||||||||||||||||||||||||
(defn- simulator-add-returns [simulator invokes] (update-in simulator [:returns] into invokes)) | ||||||||||||||||||||||||||||||||||||||||
(defn- simulator-get-returns [simulator] (get-in simulator [:returns])) | ||||||||||||||||||||||||||||||||||||||||
(defn- simulator-clear-returns [simulator] (assoc-in simulator [:returns] #{})) | ||||||||||||||||||||||||||||||||||||||||
(defn- simulator-add-explicit-invokes [simulator invokes] (update-in simulator [:explicit-invokes] into invokes)) | ||||||||||||||||||||||||||||||||||||||||
(defn- simulator-get-explicit-invokes [simulator] (get-in simulator [:explicit-invokes])) | ||||||||||||||||||||||||||||||||||||||||
(defn- simulator-clear-explicit-invokes [simulator] (assoc-in simulator [:explicit-invokes] #{})) | ||||||||||||||||||||||||||||||||||||||||
(defn- simulator-add-implicit-invokes [simulator invokes] (update-in simulator [:implicit-invokes] into invokes)) | ||||||||||||||||||||||||||||||||||||||||
(defn- simulator-get-implicit-invokes [simulator] (get-in simulator [:implicit-invokes])) | ||||||||||||||||||||||||||||||||||||||||
(defn- simulator-clear-implicit-invokes [simulator] (assoc-in simulator [:implicit-invokes] #{})) | ||||||||||||||||||||||||||||||||||||||||
(defn- simulator-add-component-invokes [simulator invokes] (update-in simulator [:component-invokes] into invokes)) | ||||||||||||||||||||||||||||||||||||||||
(defn- simulator-get-component-invokes [simulator] (get-in simulator [:component-invokes])) | ||||||||||||||||||||||||||||||||||||||||
(defn- simulator-clear-component-invokes [simulator] (assoc-in simulator [:component-invokes] #{})) | ||||||||||||||||||||||||||||||||||||||||
(defn- simulator-add-invoke-paths [simulator invokes] (update-in simulator [:invoke-paths] into invokes)) | ||||||||||||||||||||||||||||||||||||||||
(defn- simulator-get-invoke-paths [simulator] (get-in simulator [:invoke-paths])) | ||||||||||||||||||||||||||||||||||||||||
(defn- simulator-clear-invoke-paths [simulator] (assoc-in simulator [:invoke-paths] #{})) | ||||||||||||||||||||||||||||||||||||||||
implicit control flow helpers | ||||||||||||||||||||||||||||||||||||||||
filter methods that contain implicit control flow invokes | (defn filter-implicit-cf-invoke-methods [methods] (->> methods (filter (fn [^SootMethod method] (->> [method] mapcat-invoke-methodrefs (filter implicit-cf?) not-empty))))) | |||||||||||||||||||||||||||||||||||||||
test whether class possibly contains implicit cf | (defn implicit-cf-class? [class] (->> class get-transitive-implicit-cf-super-class-and-interface not-empty)) | |||||||||||||||||||||||||||||||||||||||
test whether methodref is possibly an implicit cf | (def implicit-cf? get-implicit-cf-root-class-names) | |||||||||||||||||||||||||||||||||||||||
(defn implicit-cf-task? [method] (set/intersection (->> method get-implicit-cf-root-class-names) (set (->> implicit-cf-marker-task keys)))) | ||||||||||||||||||||||||||||||||||||||||
(defn implicit-cf-component? [method] (set/intersection (->> method get-implicit-cf-root-class-names) (set (->> implicit-cf-marker-component keys)))) | ||||||||||||||||||||||||||||||||||||||||
(defn get-transitive-implicit-cf-super-class-and-interface [class] (set/intersection (set (keys implicit-cf-marker)) (->> class get-transitive-super-class-and-interface (map get-soot-class-name) set))) | ||||||||||||||||||||||||||||||||||||||||
(defn get-implicit-cf-root-class-names [method] (let [class (->> method get-soot-class) name (->> method get-soot-name)] (->> (get-transitive-implicit-cf-super-class-and-interface class) (filter #(let [t (get implicit-cf-marker %)] (or (= t :all) (contains? t name)))) not-empty))) | ||||||||||||||||||||||||||||||||||||||||
domain knowledge | ||||||||||||||||||||||||||||||||||||||||
(def ^:private implicit-cf-marker-task {"java.lang.Thread" #{"start"} "java.lang.Runnable" #{"run"} "java.util.concurrent.Callable" #{"call"} "java.util.concurrent.Executor" #{"execute"} "java.util.concurrent.ExecutorService" #{"submit" "execute"} "java.lang.Class" #{"forName" "getMethod" "getField"} "java.lang.reflect.Method" #{"invoke"} "java.lang.reflect.Field" :all "android.os.Handler" #{"post" "postAtFrontOfQueue" "postAtTime" "postDelayed"}}) | ||||||||||||||||||||||||||||||||||||||||
(def ^:private implicit-cf-marker-component {"android.content.Context" #{"startActivity" "startActivities" "startService" "stopService" "bindService" "unbindService" "sendBroadcast" "sendBrocastAsUser" "sendOrderedBroadcast" "sendOrderedBroadcastAsUser" "sendStickyBroadcast" "sendStickyBroadcastAsUser" "registerComponentCallbacks" "registerReceiver"}}) | ||||||||||||||||||||||||||||||||||||||||
these methods mark implicit control flows | (def ^:private implicit-cf-marker (merge implicit-cf-marker-task implicit-cf-marker-component)) | |||||||||||||||||||||||||||||||||||||||
safe classes are the ones that can be simulated in Clojure | (def ^:private safe-invokes {;;; java.lang ;; interface "java.lang.Iterable" :all ;; classes "java.lang.String" :all "java.lang.StringBuilder" :all "java.lang.StringBuffer" :all "java.lang.Math" :all "java.lang.StrictMath" :all "java.lang.Integer" :all "java.lang.Long" :all "java.lang.Double" :all "java.lang.Float" :all "java.lang.Byte" :all "java.lang.Character" :all "java.lang.Short" :all "java.lang.Boolean" :all "java.lang.Void" :all "java.lang.System" #{"nanoTime" "currentTimeMillis"} ;;; java.util ;; interface "java.util.Collection" :all "java.util.Comparator" :all "java.util.Deque" :all "java.util.Enumeration" :all "java.util.Formattable" :all "java.util.Iterator" :all "java.util.List" :all "java.util.ListIterator" :all "java.util.Map" :all "java.util.Map$Entry" :all "java.util.NavigableMap" :all "java.util.NavigableSet" :all "java.util.Queue" :all "java.util.RandomAccess" :all "java.util.Set" :all "java.util.SortedMap" :all "java.util.SortedSet" :all ;; classes "java.util.ArrayList" :all "java.util.ArrayDeque" :all "java.util.Arrays" :all "java.util.BitSet" :all "java.util.Calendar" :all "java.util.Collections" :all "java.util.Currency" :all "java.util.Date" :all "java.util.Dictionary" :all "java.util.EnumMap" :all "java.util.EnumSet" :all "java.util.Formatter" :all "java.util.GregorianCalendar" :all "java.util.HashMap" :all "java.util.HashSet" :all "java.util.Hashtable" :all "java.util.IdentityHashMap" :all "java.util.LinkedHashMap" :all "java.util.LinkedHashSet" :all "java.util.LinkedList" :all "java.util.Locale" :all "java.util.Locale$Builder" :all "java.util.Objects" :all "java.util.PriorityQueue" :all "java.util.Properties" :all "java.util.Random" :all "java.util.SimpleTimeZone" :all "java.util.Stack" :all "java.util.StringTokenizer" :all "java.util.TreeMap" :all "java.util.TreeSet" :all "java.util.UUID" :all "java.util.Vector" :all "java.util.WeakHashMap" :all}) | |||||||||||||||||||||||||||||||||||||||
(ns woa.apk.dex.soot.util ;; internal libs (:use woa.util) ;; common libs (:require [clojure.string :as str] [clojure.set :as set] [clojure.walk :as walk] [clojure.zip :as zip] [clojure.java.io :as io] [clojure.pprint :refer [pprint print-table]] [clojure.stacktrace :refer [print-stack-trace]]) ;; import (:import (soot G G$GlobalObjectGetter PhaseOptions PackManager Scene Pack Unit SootClass SootMethod SootMethodRef) (soot.options Options) (soot.jimple Stmt))) | ||||||||||||||||||||||||||||||||||||||||
declaration | ||||||||||||||||||||||||||||||||||||||||
(declare soot-queryable?) (declare find-method-candidates) (declare get-application-classes get-application-methods) (declare get-method-body map-class-bodies run-body-packs) (declare mapcat-invoke-methodrefs resolve-methodrefs mapcat-invoke-methods) (declare get-transitive-super-class-and-interface get-interesting-transitive-super-class-and-interface transitive-ancestor?) (declare filter-interesting-methods) (declare get-cg mapcat-edgeout-methods) (declare android-api?) (declare with-soot new-g-objgetter) (declare mute unmute with-silence) | ||||||||||||||||||||||||||||||||||||||||
implementation | ||||||||||||||||||||||||||||||||||||||||
SootQuery | ||||||||||||||||||||||||||||||||||||||||
(defprotocol SootQuery (get-soot-class [this]) (get-soot-class-name [this]) (get-soot-name [this]) (soot-resolve [this])) | ||||||||||||||||||||||||||||||||||||||||
test whether SottQuery can be applied on cand without Exception | (defn soot-queryable? [cand] (try (let [class (-> cand get-soot-class)] (-> cand get-soot-name) (-> cand get-soot-class-name) (.. class getPackageName) true) (catch Exception e false))) | |||||||||||||||||||||||||||||||||||||||
(extend-type nil SootQuery (get-soot-class [this] nil) (get-soot-class-name [this] nil) (get-soot-name [this] nil) (soot-resolve [this] nil)) | ||||||||||||||||||||||||||||||||||||||||
(extend-type soot.SootClass SootQuery (get-soot-class [this] this) (get-soot-class-name [this] (get-soot-name this)) (get-soot-name [this] (.. this getName)) (soot-resolve [this] this)) | ||||||||||||||||||||||||||||||||||||||||
(extend-type soot.SootMethod SootQuery (get-soot-class [this] (.. this getDeclaringClass)) (get-soot-class-name [this] (->> this get-soot-class get-soot-name)) (get-soot-name [this] (.. this getName)) (soot-resolve [this] this)) | ||||||||||||||||||||||||||||||||||||||||
(extend-type soot.SootMethodRef SootQuery (get-soot-class [this] (.. this declaringClass)) (get-soot-class-name [this] (->> this get-soot-class get-soot-name)) (get-soot-name [this] (.. this name)) (soot-resolve [this] (.. this resolve))) | ||||||||||||||||||||||||||||||||||||||||
(extend-type soot.SootField SootQuery (get-soot-class [this] (.. this getDeclaringClass)) (get-soot-class-name [this] (->> this get-soot-class get-soot-name)) (get-soot-name [this] (.. this getName)) (soot-resolve [this] this)) | ||||||||||||||||||||||||||||||||||||||||
(extend-type soot.SootFieldRef SootQuery (get-soot-class [this] (.. this declaringClass)) (get-soot-class-name [this] (->> this get-soot-class get-soot-name)) (get-soot-name [this] (.. this name)) (soot-resolve [this] (.. this resolve))) | ||||||||||||||||||||||||||||||||||||||||
(extend-type String SootQuery (get-soot-class [this] (.. (Scene/v) (getSootClass this))) (get-soot-class-name [this] this) (get-soot-name [this] this) (soot-resolve [this] ;; only Class string can be reasonably resolved (get-soot-class this))) | ||||||||||||||||||||||||||||||||||||||||
(extend-type soot.jimple.ClassConstant SootQuery (get-soot-class [this] (->> (.. this getValue) get-soot-class)) (get-soot-class-name [this] (.. this getValue)) (get-soot-name [this] (->> this get-soot-class-name)) (soot-resolve [this] (->> this get-soot-class))) | ||||||||||||||||||||||||||||||||||||||||
(extend-type soot.RefType SootQuery (get-soot-class [this] (.. this getSootClass)) (get-soot-class-name [this] (.. this getClassName)) (get-soot-name [this] (->> this get-soot-class-name)) (soot-resolve [this] (->> this get-soot-class))) | ||||||||||||||||||||||||||||||||||||||||
(extend-type soot.ArrayType SootQuery (get-soot-class [this] (->> (.. this getArrayElementType) get-soot-class)) (get-soot-class-name [this] (->> this get-soot-class get-soot-name)) (get-soot-name [this] (->> this get-soot-class-name)) (soot-resolve [this] (->> this get-soot-class))) | ||||||||||||||||||||||||||||||||||||||||
(extend-type soot.jimple.FieldRef SootQuery (get-soot-class [this] (->> (.. this getFieldRef) get-soot-class)) (get-soot-class-name [this] (->> this get-soot-class get-soot-name)) (get-soot-name [this] (->> (.. this getFieldRef) get-soot-name)) (soot-resolve [this] (.. this getField))) | ||||||||||||||||||||||||||||||||||||||||
Soot Method helper | ||||||||||||||||||||||||||||||||||||||||
args=nil: all method of the-class with method-name; otherwise: match by argument numbers | (defn find-method-candidates [the-class method-name args] (when-let [methods (not-empty (->> (.. the-class getMethods) (filter #(= (->> % get-soot-name) method-name))))] (cond (nil? args) methods :otherwise (->> methods (filter #(= (if (number? args) args (count args)) (.. ^SootMethod % getParameterCount))))))) | |||||||||||||||||||||||||||||||||||||||
Soot Class helpers | ||||||||||||||||||||||||||||||||||||||||
get application classes in scene | (defn get-application-classes [scene] (->> (.. scene getApplicationClasses snapshotIterator) iterator-seq set)) | |||||||||||||||||||||||||||||||||||||||
get application methods in scene | (defn get-application-methods [scene] (->> scene get-application-classes (mapcat #(try (.. ^SootClass % getMethods) (catch Exception e nil))) set)) | |||||||||||||||||||||||||||||||||||||||
Soot Body helpers | ||||||||||||||||||||||||||||||||||||||||
get method body | (defn get-method-body [^SootMethod method] (if (.. method hasActiveBody) (.. method getActiveBody) (when (and (not (.. method isPhantom)) ;; method must have backing source (.. method getSource)) (.. method retrieveActiveBody)))) | |||||||||||||||||||||||||||||||||||||||
map classes to their method bodies | (defn map-class-bodies [classes] (->> classes (remove #(.. ^SootClass % isPhantom)) (mapcat #(->> (.. ^SootClass % getMethods) seq (map get-method-body) (filter identity))))) | |||||||||||||||||||||||||||||||||||||||
run body packs over application classes | (defn run-body-packs [& {:keys [scene pack-manager body-packs verbose]}] (doto scene (.loadNecessaryClasses)) ;; force application class bodies to be mapped at least once (let [bodies (->> scene get-application-classes map-class-bodies) packs (->> body-packs (map #(.. ^PackManager pack-manager (getPack ^String %))))] (doseq [^Pack pack packs] (when pack (doseq [^SootBody body bodies] (try (.. pack (apply body)) ;; catch Exception to prevent it destroys outer threads (catch Exception e (print-stack-trace-if-verbose e verbose)))))))) | |||||||||||||||||||||||||||||||||||||||
invoker-invokee relationship helpers | ||||||||||||||||||||||||||||||||||||||||
phantom SootClass has SootMethodRef but not SootMethod | ||||||||||||||||||||||||||||||||||||||||
mapcat methods to their invoked methodrefs | (defn mapcat-invoke-methodrefs [methods] (->> methods (remove #(.. ^SootMethod % isPhantom)) ;; try retrieveActiveBody (filter #(try (.. ^SootMethod % retrieveActiveBody) true (catch Exception e false))) (mapcat #(iterator-seq (.. ^SootMethod % retrieveActiveBody getUnits snapshotIterator))) (filter #(.. ^Stmt % containsInvokeExpr)) (map #(.. ^Stmt % getInvokeExpr getMethodRef)))) | |||||||||||||||||||||||||||||||||||||||
mapcat methods to their invoked methods | (defn mapcat-invoke-methods [methods] (->> methods mapcat-invoke-methodrefs ;; deduplication early set resolve-methodrefs set)) | |||||||||||||||||||||||||||||||||||||||
resolve methodrefs | (defn resolve-methodrefs [methodrefs] (->> methodrefs (remove #(.. ^SootMethodRef % declaringClass isPhantom)) (filter #(try (.. ^SootMethodRef % resolve) true (catch Exception e false))) (map #(.. ^SootMethodRef % resolve)))) | |||||||||||||||||||||||||||||||||||||||
interesting method helpers | ||||||||||||||||||||||||||||||||||||||||
filter interesting methodrefs | (defn filter-interesting-methods [interesting-method? methods] (->> methods (filter interesting-method?))) | |||||||||||||||||||||||||||||||||||||||
Soot callgraph helpers | ||||||||||||||||||||||||||||||||||||||||
get Call Graph from scene | (defn get-cg [scene] (when (.. scene hasCallGraph) (.. scene getCallGraph))) | |||||||||||||||||||||||||||||||||||||||
mapcat methods to their edgeout methods on cg | (defn mapcat-edgeout-methods [methods cg] (when cg (->> methods (mapcat #(iterator-seq (.. ^soot.jimple.toolkits.callgraph.CallGraph cg (edgesOutOf %)))) (map #(.. ^soot.jimple.toolkits.callgraph.Edge % getTgt)) set))) | |||||||||||||||||||||||||||||||||||||||
helpers | ||||||||||||||||||||||||||||||||||||||||
test see if obj is Android API | (defn android-api? [obj] (re-find #"^(android\.|com\.android\.|dalvik\.)" (-> obj get-soot-class-name))) | |||||||||||||||||||||||||||||||||||||||
Soot body wrapper | ||||||||||||||||||||||||||||||||||||||||
Soot mutex: Soot is unfortunately Singleton | (def soot-mutex (Object.)) | |||||||||||||||||||||||||||||||||||||||
System's exsiting security manager | (def system-security-manager (System/getSecurityManager)) | |||||||||||||||||||||||||||||||||||||||
prevent Soot brining down the system with System.exit | (def noexit-security-manager ;; http://stackoverflow.com/questions/21029651/security-manager-in-clojure/21033599#21033599 (proxy [SecurityManager] [] (checkPermission ([^java.security.Permission perm] (when (.startsWith (.getName perm) "exitVM") (throw (SecurityException. "no exit for Soot")))) ([^java.security.Permission perm ^Object context] (when (.startsWith (.getName perm) "exitVM") (throw (SecurityException. "no exit for Soot"))))))) | |||||||||||||||||||||||||||||||||||||||
get transitive super class and interfaces known to Soot this memoized function is initilized in with-soot | (def get-transitive-super-class-and-interface nil) | |||||||||||||||||||||||||||||||||||||||
get interesting transitive super class and interfaces known to Soot this memoized function is initilized in with-soot | (def get-interesting-transitive-super-class-and-interface nil) | |||||||||||||||||||||||||||||||||||||||
name-or-class-a is a transitive ancestor (super class/interface) of class-b this memoized function is initilized in with-soot | (def transitive-ancestor? nil) | |||||||||||||||||||||||||||||||||||||||
create a new Soot context (G$GlobalObjectGetter) | (defn new-g-objgetter [] (let [g (new G)] (reify G$GlobalObjectGetter (getG [this] g) (reset [this])))) | |||||||||||||||||||||||||||||||||||||||
wrap body with major Soot refs at the call time: g, scene, pack-manager, options, phase-options; g can be (optionally) provided with g-objgetter (nil to ask fetch the G at the call time); (G/reset) at the end if "reset?" is true | (defmacro with-soot [g-objgetter reset? & body] `(locking soot-mutex (let [get-transitive-super-class-and-interface# (memoize (fn [class-or-interface#] ;; preserve order (let [known# (atom []) class-or-interface# (get-soot-class class-or-interface#)] (loop [worklist# #{class-or-interface#} visited# #{}] (when-not (empty? worklist#) (let [new-worklist# (atom #{})] (doseq [item# worklist# :when (not (visited# item#))] (swap! known# conj item#) ;; interfaces (swap! new-worklist# into (->> (.. item# getInterfaces snapshotIterator) iterator-seq)) ;; superclass? (when (.. item# hasSuperclass) (swap! new-worklist# conj (.. item# getSuperclass)))) (recur (set/difference @new-worklist# worklist#) (set/union visited# worklist#))))) @known#))) get-interesting-transitive-super-class-and-interface# (memoize (fn [class-or-interface# interesting?#] ;; preserve order (let [known# (atom []) class-or-interface# (get-soot-class class-or-interface#)] (loop [worklist# #{class-or-interface#} visited# #{}] (when-not (empty? worklist#) (let [new-worklist# (atom #{})] (doseq [item# worklist# :when (not (visited# item#))] (if (interesting?# item#) ;; found the most close interesting ancestor: do not follow its ancestors (swap! known# conj item#) ;; otherwise, follow its ancestors (do ;; interfaces (swap! new-worklist# into (->> (.. item# getInterfaces snapshotIterator) iterator-seq)) ;; superclass? (when (.. item# hasSuperclass) (swap! new-worklist# conj (.. item# getSuperclass)))))) (recur (set/difference @new-worklist# worklist#) (set/union visited# worklist#))))) @known#))) transitive-ancestor?# (memoize (fn [name-or-class-a# class-b#] (contains? (->> class-b# get-transitive-super-class-and-interface (map #(.. ^SootClass % getName)) set) (if (instance? SootClass name-or-class-a#) (.. name-or-class-a# getName) (str name-or-class-a#))))) soot-init# (fn [] ;; set up memoize functions so that they won't retain objects across (alter-var-root #'get-transitive-super-class-and-interface (fn [_#] get-transitive-super-class-and-interface#)) (alter-var-root #'get-interesting-transitive-super-class-and-interface (fn [_#] get-interesting-transitive-super-class-and-interface#)) (alter-var-root #'transitive-ancestor? (fn [_#] transitive-ancestor?#))) ;; we have to use this instead of clean# due to the use in ~(when reset? ...) ~'_soot-clean_ (fn [] (alter-var-root #'get-transitive-super-class-and-interface (constantly nil)) (alter-var-root #'transitive-ancestor? (constantly nil)) (G/setGlobalObjectGetter nil))] (try (soot-init#) (System/setSecurityManager noexit-security-manager) (when (instance? G$GlobalObjectGetter ~g-objgetter) (G/setGlobalObjectGetter ~g-objgetter)) (let [~'soot-g (G/v) ~'soot-scene (Scene/v) ~'soot-pack-manager (PackManager/v) ~'soot-options (Options/v) ~'soot-phase-options (PhaseOptions/v)] ~@body ~(when reset? `(~'_soot-clean_))) (catch Exception e# ;; reset Soot state (~'_soot-clean_) (throw e#)) (finally (System/setSecurityManager system-security-manager)))))) | |||||||||||||||||||||||||||||||||||||||
mutter | (def ^:private mutter (java.io.PrintStream. (proxy [java.io.OutputStream] [] (write [_ _1 _2])))) (def ^:private original-system-out System/out) | |||||||||||||||||||||||||||||||||||||||
execute the body in silence | (defmacro with-silence [& body] `(try (mute) ~@body (finally (unmute)))) | |||||||||||||||||||||||||||||||||||||||
no output | (defn mute [] (set! (. (G/v) out) mutter) (System/setOut mutter)) | |||||||||||||||||||||||||||||||||||||||
allow output again | (defn unmute [] (System/setOut original-system-out) (set! (. (G/v) out) original-system-out)) | |||||||||||||||||||||||||||||||||||||||
(ns woa.apk.parse ;; internal libs (:require [woa.apk.dex.parse :refer [get-the-dex-sha256-digest]] [woa.apk.aapt.parse :refer [get-manifest]] [woa.apk.util :refer [get-apk-cert-sha256-digest get-file-sha256-digest]]) ;; common libs (:require [clojure.string :as str] [clojure.set :as set] [clojure.walk :as walk] [clojure.zip :as zip] [clojure.java.io :as io] [clojure.pprint :refer [pprint print-table]] [clojure.stacktrace :refer [print-stack-trace]])) | ||||||||||||||||||||||||||||||||||||||||
declaration | (declare parse-apk) | |||||||||||||||||||||||||||||||||||||||
implementation | ||||||||||||||||||||||||||||||||||||||||
parse apk: the common part | (defn parse-apk [apk-name] {:manifest (get-manifest apk-name) :dex-sha256 (get-the-dex-sha256-digest apk-name) :cert-sha256 (get-apk-cert-sha256-digest apk-name) :sha256 (get-file-sha256-digest apk-name)}) | |||||||||||||||||||||||||||||||||||||||
(ns woa.apk.util ;; internal libs ;; common libs (:require [clojure.string :as str] [clojure.set :as set] [clojure.walk :as walk] [clojure.zip :as zip] [clojure.java.io :as io] [clojure.pprint :refer [pprint print-table]] [clojure.stacktrace :refer [print-stack-trace]]) ;; special libs (:require [pandect.algo.sha256 :refer [sha256-bytes sha256]] [clojure.java.shell :refer [sh]]) ;; imports ;; http://stackoverflow.com/a/1802126 (:import (java.io ByteArrayInputStream)) ;; http://stackoverflow.com/a/5419767 (:import (java.util.zip ZipFile ZipInputStream ZipEntry)) ;; http://stackoverflow.com/a/19194580 (:import (java.nio.file Files Paths StandardCopyOption)) ;; http://stackoverflow.com/a/1264756 (:import (org.apache.commons.io IOUtils))) | ||||||||||||||||||||||||||||||||||||||||
(declare get-apk-file-bytes get-apk-file-input-stream extract-apk-file get-apk-file-sha256-digest get-apk-cert-sha256-digest) | ||||||||||||||||||||||||||||||||||||||||
get bytes of file-name in apk | (defn ^bytes get-apk-file-bytes [apk file-name] (with-open [apk (ZipFile. ^String apk)] (IOUtils/toByteArray ^java.io.InputStream (.getInputStream apk (.getEntry apk file-name))))) | |||||||||||||||||||||||||||||||||||||||
get an input stream of file-name in apk | (defn ^java.io.InputStream get-apk-file-input-stream [apk file-name] (ByteArrayInputStream. (get-apk-file-bytes apk file-name))) | |||||||||||||||||||||||||||||||||||||||
extract file-name in apk to output-file-name | (defn extract-apk-file [apk file-name output-file-name] (Files/copy ^java.io.InputStream (get-apk-file-input-stream apk file-name) (Paths/get output-file-name (into-array String [""])) (into-array StandardCopyOption [StandardCopyOption/REPLACE_EXISTING]))) | |||||||||||||||||||||||||||||||||||||||
get sha256 digest of file-name in apk | (defn get-apk-file-sha256-digest [apk file-name] (sha256 (get-apk-file-bytes apk file-name))) | |||||||||||||||||||||||||||||||||||||||
get sha256 digest of apk's cert | (defn get-apk-cert-sha256-digest [apk] (let [raw (:out (sh "keytool" "-printcert" "-jarfile" apk)) [_ digest] (re-find #"SHA256:\s+(\S+)" raw)] digest)) | |||||||||||||||||||||||||||||||||||||||
get sha256 digest of the-file | (defn get-file-sha256-digest [the-file] (sha256 (io/file the-file))) | |||||||||||||||||||||||||||||||||||||||
(ns woa.core ;; internal libs (:require [woa.util :refer [print-stack-trace-if-verbose]] [woa.core.signature :refer [compute-cgdfd-signature compute-cgdfd]] [woa.apk.parse :as apk-parse] [woa.apk.aapt.parse :as aapt-parse] [woa.neo4j.core :as neo4j] [woa.apk.dex.soot.parse :as soot-parse] [woa.virustotal.core :as vt]) ;; common libs (:require [clojure.string :as str] [clojure.set :as set] [clojure.walk :as walk] [clojure.zip :as zip] [clojure.java.io :as io] [clojure.pprint :refer [pprint print-table]] [clojure.stacktrace :refer [print-stack-trace]]) ;; special libs (:require [clojure.tools.cli :refer [parse-opts]]) (:require [clojure.java.shell :refer [sh]]) (:require [me.raynes.fs :as fs]) (:require [clojure.tools.nrepl.server :refer [start-server stop-server]]) (:require [cider.nrepl :refer [cider-nrepl-handler]]) (:require [taoensso.nippy :as nippy]) ;; imports (:import (java.util.concurrent Executors TimeUnit)) ;; config (:gen-class)) | ||||||||||||||||||||||||||||||||||||||||
(def cli-options [ ;; general options ["-h" "--help" "you are reading it now"] ["-v" "--verbose" "be verbose (more \"v\" for more verbosity)" :default 0 :assoc-fn (fn [m k _] (update-in m [k] inc))] ["-L" "--no-line-reading" "do not read from stdin; exit if other tasks complete"] ["-i" "--interactive" "do not exit (i.e., shutdown-agents) at the end"] [nil "--delay-start SEC" "delay start for a random seconds from 1 to (max) SEC" :parse-fn #(Integer/parseInt %) :default 0] ;; nREPL config [nil "--nrepl-port PORT" "REPL port" :parse-fn #(Integer/parseInt %) :validate [#(< 0 % 0x10000) (format "Must be a number between 0 and %d (exclusively)" 0x10000)]] ;; prepations [nil "--prep-tags TAGS" "TAGS is a Clojure vector of pairs of label types to properties, e.g., [[[\"Dataset\"] {\"id\" \"dataset-my\" \"name\" \"my dataset\"}]]"] [nil "--prep-virustotal" "obtain VirusTotal tags"] [nil "--virustotal-apikey APIKEY" "VirusTotal API key"] [nil "--virustotal-rate-limit LIMIT-PER-MIN" "number of maximal API calls per minute" :parse-fn #(Integer/parseInt %) :default 4] [nil "--virustotal-backoff SEC" "number of seconds to backoff when exceeding rate limit" :parse-fn #(Integer/parseInt %) :default 5] [nil "--virustotal-submit" "whether submit sample to VirusTotal if not found"] ;; Soot config ["-s" "--soot-task-build-model" "build APK model with Soot"] [nil "--soot-android-jar-path PATH" "path of android.jar for Soot's Dexpler"] [nil "--soot-basic-block-simulation-budget BUDGET" "basic block simulation budget" :parse-fn #(Long/parseLong %) :default 50] [nil "--soot-method-simulation-depth-budget BUDGET" "method invocation simulation budget" :parse-fn #(Long/parseLong %) :default 10] [nil "--soot-simulation-collection-size-budget BUDGET" "array size simulation budget" :parse-fn #(Long/parseLong %) :default 10000] [nil "--soot-simulation-conservative-branching" "branching based on conditions: more precision at the cost of less coverage before budget depletion."] [nil "--soot-simulation-linear-scan" "do not branch or loop: more coverage at the cost of precision"] ["-j" "--soot-parallel-jobs JOBS" "number of concurrent threads for analyzing methods" :parse-fn #(Integer/parseInt %) :default 1 :validate [#(> % 0) (format "You need at least 1 job to proceed")]] [nil "--soot-show-result" "show APK analysis result"] [nil "--soot-no-implicit-cf" "do not detect implicit control flows (for comparison)"] [nil "--soot-dump-all-invokes" "dump all invokes"] [nil "--soot-result-exclude-app-methods" "exclude app internal methods from the result"] [nil "--soot-debug-show-each-statement" "debug facility: show each processed statement"] [nil "--soot-debug-show-locals-per-statement" "debug facility: show locals per each statement"] [nil "--soot-debug-show-all-per-statement" "debug facility: show all per each statement"] [nil "--soot-debug-show-implicit-cf" "debug facility: show all implicit control flows"] [nil "--soot-debug-show-safe-invokes" "debug facility: show all safe invokes"] [nil "--soot-debug-show-exceptions" "debug facility: show all exceptions"] ["-d" "--dump-model FILE" "dump binary APK model; append dump file paths to FILE"] ["-O" "--overwrite-model" "overwrite model while dumping"] ["-l" "--load-model FILE" "load binary APK model; load from dump file paths in FILE"] ["-c" "--convert-model" "convert model between binary and readable formats"] [nil "--readable-model" "dump/load readable APK model; --dump/load-model FILE will dump/load readable model directly to/from FILE"] [nil "--println-model" "println loaded APK model"] [nil "--pprint-model" "pprint loaded APK model"] [nil "--debug-cgdfd" "debug facility: show cgdfd and signature"] ;; Neo4j config [nil "--neo4j-port PORT" "Neo4j server port" :parse-fn #(Integer/parseInt %) :default 7474 :validate [#(< 0 % 0x10000) (format "Must be a number between 0 and %d (exclusively)" 0x10000)]] [nil "--neo4j-protocol PROTOCOL" "Neo4j server protocol (http/https)" :default "http"] [nil "--neo4j-conn-backoff SEC" "Neo4j connection retry max random backoff in seconds" :parse-fn #(Integer/parseInt %) :default 60] ;; Neo4j tasks ["-n" "--neo4j-task-populate" "populate Neo4j with APK model"] ["-t" "--neo4j-task-tag" "tag Neo4j Apk nodes with labels"] ["-T" "--neo4j-task-untag" "untag Neo4j Apk nodes with labels"] ["-g" "--neo4j-task-add-callback-signature" "add Neo4j CallbackSignature nodes"] ["-G" "--neo4j-task-remove-callback-signature" "remove Neo4j CallbackSignature nodes"] [nil "--neo4j-include-methodinstance" "include MethodInstance in the WoA model"] [nil "--neo4j-no-callgraph" "not include call graph (CG) in the WoA model"] ["-D" "--neo4j-dump-model-batch-csv PREFIX" "dump Neo4j batch import CSV files to PREFIX; ref: https://github.com/jexp/batch-import/tree/2.1"] ;; misc tasks [nil "--dump-manifest" "dump AndroidManifest.xml"] ]) | ||||||||||||||||||||||||||||||||||||||||
for consumption by nREPL session | (def main-options (atom nil)) | |||||||||||||||||||||||||||||||||||||||
establish critical section | (def mutex (Object.)) | |||||||||||||||||||||||||||||||||||||||
completed task counter | (def completed-task-counter (atom 0)) | |||||||||||||||||||||||||||||||||||||||
synchronize verbose ouput | (defmacro with-mutex-locked [& body] `(locking mutex ~@body)) | |||||||||||||||||||||||||||||||||||||||
(defn- debug-print-cgdfd [apk] (let [result (atom {})] (let [the-dex (:dex apk)] (dorun (for [comp-package-name (->> the-dex keys)] (dorun (for [comp-class-name (->> (get-in the-dex [comp-package-name]) keys)] (dorun (for [callback-name (->> (get-in the-dex [comp-package-name comp-class-name :callbacks]) keys)] (let [invoke-paths (get-in the-dex [comp-package-name comp-class-name :callbacks callback-name :invoke-paths]) cgdfd (compute-cgdfd invoke-paths) signature (compute-cgdfd-signature cgdfd)] (swap! result assoc [comp-package-name comp-class-name callback-name] {:cgdfd cgdfd :signature signature}))))))))) (doseq [k (keys @result)] (swap! result update-in [k] (fn [old] (update-in old [:cgdfd] #(->> % (sort-by first) vec))))) (swap! result assoc :apk (:sha256 apk)) (pprint @result))) | ||||||||||||||||||||||||||||||||||||||||
do the real work on apk | (defn work [{:keys [file-path tags] :as task} {:keys [verbose soot-task-build-model dump-model overwrite-model readable-model debug-cgdfd neo4j-port neo4j-protocol neo4j-task-populate neo4j-task-tag neo4j-task-untag neo4j-task-add-callback-signature neo4j-task-remove-callback-signature dump-manifest] :as options}] (when (and file-path (fs/readable? file-path)) (when (and verbose (> verbose 1)) (println "processing" file-path)) (let [start-time (System/currentTimeMillis)] (try (when dump-manifest (print (aapt-parse/get-manifest-xml file-path)) (flush)) (when soot-task-build-model (let [apk (apk-parse/parse-apk file-path) dump-fname (str (:sha256 apk) ".model-dump")] (when (or overwrite-model (not (and dump-model (fs/exists? dump-fname) (do (when verbose (println dump-fname "exists: skipped;" "overwrite with \"--overwrite-model\"")) true)))) (let [apk (soot-parse/parse-apk file-path (merge options ;; piggyback layout-callbacks on options {:layout-callbacks (aapt-parse/get-layout-callbacks file-path)}))] (when dump-model (try (with-open [model-io (io/writer dump-model :append true)] (binding [*out* model-io] (if readable-model (prn apk) (with-open [model-io (io/output-stream dump-fname)] (nippy/freeze-to-out! (java.io.DataOutputStream. model-io) apk) ;; write the dump file name out (println dump-fname))))) (catch Exception e (print-stack-trace-if-verbose e verbose)))) (when debug-cgdfd (debug-print-cgdfd apk)) (when neo4j-task-populate (neo4j/populate-from-parsed-apk apk options)) (cond neo4j-task-add-callback-signature (try (neo4j/add-callback-signature apk options) (catch Exception e (print-stack-trace-if-verbose e verbose))) neo4j-task-remove-callback-signature (try (neo4j/remove-callback-signature apk options) (catch Exception e (print-stack-trace-if-verbose e verbose)))))))) (let [apk (apk-parse/parse-apk file-path)] (cond neo4j-task-tag (neo4j/tag-apk apk tags options) neo4j-task-untag (neo4j/untag-apk apk tags options))) (when (and verbose (> verbose 0)) (with-mutex-locked (swap! completed-task-counter inc) (println (format "%1$d: %2$s processed in %3$.3f seconds" @completed-task-counter file-path (/ (- (System/currentTimeMillis) start-time) 1000.0))))) (catch Exception e (print-stack-trace-if-verbose e verbose)))))) | |||||||||||||||||||||||||||||||||||||||
main entry | (defn -main [& args] (let [raw (parse-opts args cli-options) {:keys [options summary errors]} raw {:keys [verbose interactive delay-start help no-line-reading prep-tags prep-virustotal virustotal-rate-limit virustotal-backoff virustotal-submit nrepl-port load-model dump-model convert-model println-model pprint-model readable-model debug-cgdfd neo4j-task-populate neo4j-task-add-callback-signature neo4j-task-remove-callback-signature neo4j-dump-model-batch-csv]} options] (try ;; print out error messages if any (when errors (binding [*out* *err*] (doseq [error errors] (println error)))) ;; whether help is requested (cond help (do (println "<BUILDINFO>") (println summary)) (or prep-tags prep-virustotal) (do ;; for API rate limit (let [vt-api-call-counter (atom virustotal-rate-limit) vt-start-time (atom (System/currentTimeMillis))] (loop [line (read-line)] (when line (try (prn (-> {:file-path line :tags []} (update-in [:tags] into (when (and prep-tags (not (str/blank? prep-tags))) (read-string prep-tags))) (update-in [:tags] into (when prep-virustotal (let [apk (apk-parse/parse-apk line) try-backoff (fn [] (when (<= @vt-api-call-counter 0) (let [now (System/currentTimeMillis) sleep-time (max (* virustotal-backoff 1000) (- (+ @vt-start-time (* 60 1000)) now))] (reset! vt-api-call-counter virustotal-rate-limit) (reset! vt-start-time now) (Thread/sleep sleep-time))))] (when-let [sha256 (:sha256 apk)] (try-backoff) (when-let [result (vt/get-report {:sha256 sha256} options)] (swap! vt-api-call-counter dec) (when (and verbose (> verbose 2)) (binding [*out* *err*] (println "virustotal report" result))) (let [ret (atom nil)] (cond ;; if result is a map, the result is returned (map? result) (reset! ret (vt/make-report-result-into-tags result)) (= result :status-exceed-api-limit) (try-backoff) (= result :response-not-found) (when virustotal-submit (try-backoff) (let [result (vt/submit-file {:file-content (io/file line)} options)] (when (and verbose (> verbose 2)) (binding [*out* *err*] (println "virustotal submit" result)))) (swap! vt-api-call-counter dec))) @ret)))))))) (catch Exception e (print-stack-trace-if-verbose e verbose))) (recur (read-line)))))) :otherwise (do (when (and delay-start (> delay-start 0)) (let [delay-start (rand-int delay-start)] (when (> verbose 1) (println "delay start" delay-start "seconds")) (Thread/sleep (* delay-start 1000)))) (when nrepl-port ;; use separate thread to start nREPL, so do not delay other task (.. (Thread. (fn [] (try (start-server :port nrepl-port :handler cider-nrepl-handler) (catch Exception e (when (> verbose 1) (binding [*out* *err*] (println "error: nREPL server cannot start at port" nrepl-port))))))) start)) (when neo4j-task-populate ;; "create index" only need to executed once if populate-neo4j is requested (when (> verbose 1) (with-mutex-locked (println "Neo4j:" "creating index"))) (neo4j/create-index options) (when (> verbose 1) (with-mutex-locked (println "Neo4j:" "index created")))) ;; load Soot model and populate Neo4j graph ;; single-threaded to avoid Neo4j contention (when load-model (try (let [counter (atom 0)] (with-open [model-io (io/reader load-model)] (binding [*in* model-io] (loop [line (read-line)] (when line (let [apk (try (if readable-model (read-string line) (with-open [model-io (io/input-stream line)] (nippy/thaw-from-in! (java.io.DataInputStream. model-io)))) (catch Exception e (print-stack-trace-if-verbose e verbose) nil))] (when apk (when neo4j-dump-model-batch-csv (neo4j/add-to-batch-csv apk options)) (when (and apk convert-model dump-model) (try (with-open [model-io (io/writer dump-model :append true)] (binding [*out* model-io] (if readable-model ; if --load-model is in readable format ;; convert to binary model (let [dump-fname (str (:sha256 apk) ".model-dump")] (with-open [model-io (io/output-stream dump-fname)] (nippy/freeze-to-out! (java.io.DataOutputStream. model-io) apk) ;; write the dump file name out (println dump-fname))) ;; convert to readable model (prn apk)))) (catch Exception e (print-stack-trace-if-verbose e verbose)))) ((cond pprint-model pprint println-model println ;; nop :otherwise (constantly nil)) apk) (when debug-cgdfd (debug-print-cgdfd apk)) (swap! counter inc) (when (and verbose (> verbose 0)) (println (format "%1$d:" @counter) (get-in apk [:manifest :package]) (get-in apk [:manifest :android:versionCode]) (get-in apk [:sha256]))) (when neo4j-task-populate (try (neo4j/populate-from-parsed-apk apk options) (catch Exception e (print-stack-trace-if-verbose e verbose)))) (cond neo4j-task-add-callback-signature (try (neo4j/add-callback-signature apk options) (catch Exception e (print-stack-trace-if-verbose e verbose))) neo4j-task-remove-callback-signature (try (neo4j/remove-callback-signature apk options) (catch Exception e (print-stack-trace-if-verbose e verbose))))) (recur (read-line)))))))) (when neo4j-dump-model-batch-csv (neo4j/dump-batch-csv neo4j-dump-model-batch-csv options)) (catch Exception e (print-stack-trace-if-verbose e verbose)))) ;; do the work for each line (when-not no-line-reading (loop [line (read-line)] (when line ;; ex.: {:file-path "a/b.apk" :tags [{["Dataset"] {"id" "dst-my" "name" "my dataset"}}]} ;; tags must have "id" node property (let [{:keys [file-path tags] :as task} (try (read-string line) (catch Exception e (print-stack-trace-if-verbose e verbose) nil))] (try (when (and file-path (fs/readable? file-path)) (work task options)) (catch Exception e (print-stack-trace-if-verbose e verbose))) (recur (read-line)))))) (when neo4j-task-populate (when (> verbose 1) (with-mutex-locked (println "Neo4j:" "marking Android API"))) ;; mark Android API (neo4j/mark-android-api options) (when (> verbose 1) (with-mutex-locked (println "Neo4j:" "Android API marked")))))) (when interactive ;; block when interactive is requested @(promise)) (catch Exception e (print-stack-trace-if-verbose e verbose)) (finally ;; clean-up (shutdown-agents) (when (> verbose 1) (with-mutex-locked (println "shutting down"))) (System/exit 0))))) | |||||||||||||||||||||||||||||||||||||||
(ns woa.core.invoke-path ;; internal libs (:use woa.util) ;; common libs (:require [clojure.string :as str] [clojure.set :as set] [clojure.walk :as walk] [clojure.zip :as zip] [clojure.java.io :as io] [clojure.pprint :refer [pprint print-table]] [clojure.stacktrace :refer [print-stack-trace]]) ;; special libs) | ||||||||||||||||||||||||||||||||||||||||
declaration | ||||||||||||||||||||||||||||||||||||||||
(declare invoke-path-get-invocatee-map invoke-path-get-node invoke-path-get-descendants invoke-path-get-node-name) | ||||||||||||||||||||||||||||||||||||||||
implementation | ||||||||||||||||||||||||||||||||||||||||
invocatee map is a map from nodes to their invocatees | (defn invoke-path-get-invocatee-map [invoke-paths] (let [invocatees (atom {})] (process-worklist #{invoke-paths} (fn [worklist] (let [new-worklist (atom #{})] (dorun (for [work worklist] (let [node (invoke-path-get-node work) descendants (invoke-path-get-descendants work) children (map invoke-path-get-node descendants)] (swap! invocatees update-in [node] #(->> (into %1 %2) set) children) (swap! new-worklist into descendants)))) @new-worklist))) @invocatees)) | |||||||||||||||||||||||||||||||||||||||
(defn invoke-path-get-node [invoke-paths] (cond (map? invoke-paths) (->> invoke-paths keys first) :otherwise invoke-paths)) | ||||||||||||||||||||||||||||||||||||||||
(defn invoke-path-get-descendants [invoke-paths] (cond (map? invoke-paths) (->> invoke-paths vals first) :otherwise nil)) | ||||||||||||||||||||||||||||||||||||||||
(defn invoke-path-get-node-name [node] (cond ;; Soot signature format (re-matches #"^<.+>$" node) (let [[_ class method] (re-find #"^<([^:]+):\s+\S+\s+([^(]+)\(" node)] (str class "." method)) (re-matches #"^[^<].+\[.+\]" node) (let [[_ classmethod] (re-find #"^(.+)\[" node)] classmethod) :otherwise node)) | ||||||||||||||||||||||||||||||||||||||||
(ns woa.core.signature ;; internal libs (:use woa.util) (:use woa.core.invoke-path) ;; common libs (:require [clojure.string :as str] [clojure.set :as set] [clojure.walk :as walk] [clojure.zip :as zip] [clojure.java.io :as io] [clojure.pprint :refer [pprint print-table]] [clojure.stacktrace :refer [print-stack-trace]]) ;; special libs (:require [incanter.stats :as stats])) | ||||||||||||||||||||||||||||||||||||||||
declaration | ||||||||||||||||||||||||||||||||||||||||
(declare compute-cgdfd-signature compute-cgdfd) | ||||||||||||||||||||||||||||||||||||||||
implementation | ||||||||||||||||||||||||||||||||||||||||
compute the CGDFD-based signature from the input CGDFD (replace NaN with 0) | (defn compute-cgdfd-signature [cgdfd] (try (let [total (reduce + (vals cgdfd)) cgdfd (mapcat #(let [[out-degree multiplicity] %] (repeat multiplicity out-degree)) cgdfd)] ;; filter on NaN (->> [total (stats/mean cgdfd) (stats/sd cgdfd) (stats/skewness cgdfd) (stats/kurtosis cgdfd)] (map #(let [n %] (cond (try (.isNaN n) (catch Exception e false)) 0 :otherwise n))) vec)) (catch Exception e (print-stack-trace e) nil))) | |||||||||||||||||||||||||||||||||||||||
compute CGDFD (Call Graph Degree Frequency Distribution) from the input invoke-paths that represent the CG (Call Graph) | (defn compute-cgdfd [invoke-paths] (let [invocatees (invoke-path-get-invocatee-map invoke-paths) cgdfd (atom {})] (doseq [[_ invocatees] invocatees] (swap! cgdfd update-in [(count invocatees)] #(let [now %] (cond (nil? now) 1 :otherwise (inc now))))) @cgdfd)) | |||||||||||||||||||||||||||||||||||||||
(ns woa.neo4j.core ;; internal libs (:use woa.util) (:use (woa.core invoke-path signature)) ;; common libs (:require [clojure.string :as str] [clojure.set :as set] [clojure.walk :as walk] [clojure.zip :as zip] [clojure.java.io :as io] [clojure.pprint :refer [pprint print-table]] [clojure.stacktrace :refer [print-stack-trace]]) ;; special libs (:require [clojurewerkz.neocons.rest :as nr] [clojurewerkz.neocons.rest.transaction :as ntx])) | ||||||||||||||||||||||||||||||||||||||||
(declare populate-from-parsed-apk add-to-batch-csv dump-batch-csv tag-apk untag-apk add-callback-signature remove-callback-signature create-index mark-android-api connect android-api?) | ||||||||||||||||||||||||||||||||||||||||
(def defaults (atom {:neo4j-port 7474 :neo4j-protocol "http"})) | ||||||||||||||||||||||||||||||||||||||||
add to batch csv that is to be dumped laterhttps://github.com/jexp/batch-import/tree/2.1 | (let [node-props (atom #{}) rel-props (atom #{}) node-counter (atom -1) nodes (atom {}) ; node => node-counter rels (atom {}) merge-node (fn [[labels props :as node]] (swap! node-props into (keys props)) (when-not (get @nodes node) (swap! nodes assoc node (swap! node-counter inc)))) node-updates (atom {}) update-node (fn [old-node [labels props :as new-node]] (when-let [id (get @nodes old-node)] (swap! node-props into (keys props)) ;; cache the updates (swap! node-updates assoc old-node new-node))) merge-rel (fn [node1 [labels props :as rel] node2] (let [n1 (get @nodes node1) n2 (get @nodes node2)] (when (and n1 n2) (swap! rel-props into (keys props)) (swap! rels update-in [[n1 n2]] #(->> (conj %1 %2) set) rel))))] (defn add-to-batch-csv [apk {:keys [neo4j-include-methodinstance neo4j-no-callgraph] :as options}] (let [manifest (:manifest apk) dex-sha256 (:dex-sha256 apk) cert-sha256 (:cert-sha256 apk) apk-sha256 (:sha256 apk) apk-package (:package manifest) apk-version-name (:android:versionName manifest) apk-version-code (:android:versionCode manifest) the-dex (:dex apk)] (let [signing-key [["SigningKey"] {"sha256" cert-sha256}] apk [["Apk"] {"sha256" apk-sha256 "package" apk-package "versionCode" apk-version-code "versionName" apk-version-name}] dex [["Dex"] {"sha256" dex-sha256}]] (merge-node signing-key) (merge-node apk) (merge-node dex) (merge-rel signing-key [["SIGN"] nil] apk) (merge-rel apk [["CONTAIN"] nil] dex) ;; permissions (doseq [perm (->> manifest :uses-permission (map name))] (let [n [["Permission"] {"name" perm}]] (merge-node n) (merge-rel apk [["USE"] nil] n))) (doseq [perm (->> manifest :permission (map name))] (let [n [["Permission"] {"name" perm}]] (merge-node n) (merge-rel apk [["DEFINE"] nil] n))) ;; component package and class (dorun (for [comp-package-name (->> the-dex keys)] (let [comp-package [["Package"] {"name" comp-package-name}]] (merge-node comp-package) (dorun (for [comp-class-name (->> (get-in the-dex [comp-package-name]) keys)] (let [comp-class [["Class"] {"name" comp-class-name}]] (merge-node comp-class) (merge-rel comp-package [["CONTAIN"] nil] comp-class) (merge-rel dex [["CONTAIN"] nil] comp-class) (let [{:keys [android-api-ancestors callbacks]} (->> (get-in the-dex [comp-package-name comp-class-name]))] (dorun (for [ancestor android-api-ancestors] (let [ancestor-package-name (:package ancestor) ancestor-package [["Package"] {"name" ancestor-package-name}] ancestor-class-name (:class ancestor) ancestor-class [["Class"] {"name" ancestor-class-name}]] (merge-node ancestor-package) (merge-node ancestor-class) (merge-rel ancestor-package [["CONTAIN"] nil] ancestor-class) (merge-rel comp-class [["DESCEND"] nil] ancestor-class)))) (dorun (for [callback-name (->> callbacks keys)] (let [callback [["Method" "Callback"] {"name" (str comp-class-name "." callback-name)}]] (merge-node callback) (merge-rel comp-class [["CONTAIN"] nil] callback) (let [path [comp-package-name comp-class-name :callbacks callback-name]] ;; explicit control flow (let [path (conj path :explicit)] ;; deduplication (let [callback-invokes (->> (get-in the-dex path) (map #(select-keys % [:package :class :method])) (into #{}))] (dorun (for [callback-invoke callback-invokes] (let [invoke-package-name (:package callback-invoke) invoke-package [["Package"] {"name" invoke-package-name}] invoke-class-name (:class callback-invoke) invoke-class [["Class"] {"name" invoke-class-name}] invoke-name (:method callback-invoke) invoke [["Method"] {"name" (str invoke-class-name "." invoke-name)}]] (merge-node invoke-package) (merge-node invoke-class) (merge-node invoke) (merge-rel invoke-package [["CONTAIN"] nil] invoke-class) (merge-rel invoke-class [["CONTAIN"] nil] invoke) (merge-rel callback [["EXPLICIT_INVOKE"] nil] invoke) (merge-rel invoke [["INVOKED_BY"] nil] apk))))) (when neo4j-include-methodinstance (dorun (for [callback-invoke (get-in the-dex path)] (let [invoke-class-name (:class callback-invoke) invoke-class [["Class"] {"name" invoke-class-name}] invoke-name (:method callback-invoke) args (:args callback-invoke) invoke [["Method"] {"name" invoke-name}] invoke-instance [["MethodInstance"] {"name" (str invoke-class-name "." invoke-name) "args" args}]] (merge-node invoke-instance) (merge-rel invoke-instance [["INSTANCE_OF"] nil] invoke) (merge-rel callback [["EXPLICIT_INVOKE"] nil] invoke-instance)))))) ;; implicit control flow (let [path (conj path :implicit)] ;; deduplication (let [callback-invokes (->> (get-in the-dex path) (map #(select-keys % [:package :class :method])) (into #{}))] (dorun (for [callback-invoke callback-invokes] (let [invoke-package-name (:package callback-invoke) invoke-package [["Package"] {"name" invoke-package-name}] invoke-class-name (:class callback-invoke) invoke-class [["Class"] {"name" invoke-class-name}] invoke-name (:method callback-invoke) invoke [["Method"] {"name" (str invoke-class-name "." invoke-name)}]] (merge-node invoke-package) (merge-node invoke-class) (merge-node invoke) (merge-rel invoke-package [["CONTAIN"] nil] invoke-class) (merge-rel invoke-class [["CONTAIN"] nil] invoke) (merge-rel callback [["IMPLICIT_INVOKE"] nil] invoke) (merge-rel invoke [["INVOKED_BY"] nil] apk))))) (when neo4j-include-methodinstance (dorun (for [callback-invoke (get-in the-dex path)] (let [invoke-class-name (:class callback-invoke) invoke-class [["Class"] {"name" invoke-class-name}] invoke-name (:method callback-invoke) args (:args callback-invoke) invoke [["Method"] {"name" invoke-name}] invoke-instance [["MethodInstance"] {"name" (str invoke-class-name "." invoke-name) "args" args}]] (merge-node invoke-instance) (merge-rel invoke-instance [["INSTANCE_OF"] nil] invoke) (merge-rel callback [["IMPLICIT_INVOKE"] nil] invoke-instance)))))) ;; invokes that descend from Android API (let [path (conj path :descend)] (dorun (for [[api-invoke callback-invokes] (get-in the-dex path)] (let [api-package-name (:package api-invoke) api-package [["Package"] {"name" api-package-name}] api-class-name (:class api-invoke) api-class [["Class"] {"name" api-class-name}] api-name (:method api-invoke) api [["Method"] {"name" (str api-class-name "." api-name)}]] (merge-node api-package) (merge-node api-class) (merge-node api) (merge-rel api-package [["CONTAIN"] nil] api-class) (merge-rel api-class [["CONTAIN"] nil] api) (dorun (for [callback-invoke callback-invokes] (let [invoke-package-name (:package callback-invoke) invoke-package [["Package"] {"name" invoke-package-name}] invoke-class-name (:class callback-invoke) invoke-class [["Class"] {"name" invoke-class-name}] invoke-name (:method callback-invoke) invoke [["Method"] {"name" (str invoke-class-name "." invoke-name)}]] (merge-rel invoke [["DESCEND"] nil] api)))))))) (when-not neo4j-no-callgraph (let [path (conj path :invoke-paths) invoke-paths (get-in the-dex path)] (when invoke-paths ;; link the root node to the Callback and the Apk (let [root-node (invoke-path-get-node invoke-paths) node [["CallGraphNode"] {"name" (invoke-path-get-node-name root-node) "signature" root-node "apk" apk-sha256}]] (merge-node node) (merge-rel apk [["CALLGRAPH"] nil] node) (merge-rel callback [["CALLGRAPH"] nil] node)) ;; iteratively link descendants (process-worklist #{invoke-paths} (fn [worklist] (let [new-worklist (atom #{})] (dorun (for [work worklist] (let [node (invoke-path-get-node work) descendants (invoke-path-get-descendants work) children (map invoke-path-get-node descendants) parent-node [["CallGraphNode"] {"name" (invoke-path-get-node-name node) "signature" node "apk" apk-sha256}]] (merge-node parent-node) (dorun (for [child children] (let [child-node [["CallGraphNode"] {"name" (invoke-path-get-node-name child) "signature" child "apk" apk-sha256}]] (merge-node child-node) (merge-rel parent-node [["INVOKE"] nil] child-node)))) (swap! new-worklist into descendants)))) @new-worklist)))))) ;; callback signature (let [path (conj path :invoke-paths) invoke-paths (get-in the-dex path)] (when invoke-paths (let [cgdfd (compute-cgdfd invoke-paths) signature (compute-cgdfd-signature cgdfd)] (when signature (let [signature-node [["CallbackSignature"] {"name" (str comp-class-name "." callback-name) "apk" apk-sha256 "signature:double_array" (->> signature (map str) (str/join ","))}]] (merge-node signature-node) (merge-rel apk [["CALLBACK_SIGNATURE"] nil] signature-node) (merge-rel callback [["CALLBACK_SIGNATURE"] nil] signature-node))))))))))))))))) ;; app components (doseq [comp-type [:activity :service :receiver]] (doseq [[comp-name {:keys [intent-filter-action intent-filter-category]}] (->> manifest comp-type)] (let [comp-name (name comp-name) comp [["Class"] {"name" comp-name}] new-comp [["Class" "Component" (->> comp-type name str/capitalize)] {"name" comp-name}] intent-filter-actions (map name intent-filter-action) intent-filter-categories (map name intent-filter-category)] (update-node comp new-comp) (doseq [intent-filter-action-name intent-filter-actions] (let [intent-filter-action [["IntentFilterAction"] {"name" intent-filter-action-name}]] (merge-node intent-filter-action) (merge-rel intent-filter-action [["TRIGGER"] nil] comp))) (doseq [intent-filter-category-name intent-filter-categories] (let [intent-filter-category [["IntentFilterCategory"] {"name" intent-filter-category-name}]] (merge-node intent-filter-category) (merge-rel intent-filter-category [["TRIGGER"] nil] comp))))))))) (defn dump-batch-csv [batch-csv-prefix {:keys [] :as options}] ;; do the updates (doseq [[old-node new-node] @node-updates] (when-let [id (get @nodes old-node)] (swap! nodes dissoc old-node) (swap! nodes assoc new-node id))) (with-open [out (io/writer (str batch-csv-prefix ".nodes"))] (binding [*out* out] (let [id-to-node (set/map-invert @nodes) node-props (seq @node-props)] (println (format "%1$s%2$s%3$s" (str/join "\t" node-props) (if node-props "\t" "") "l:label")) (dotimes [id (inc @node-counter)] (let [[labels props :as node] (get id-to-node id)] (println (format "%1$s%2$s%3$s" (str/join "\t" (map #(get props %) node-props)) (if node-props "\t" "") (str/join "," labels)))))))) (with-open [out (io/writer (str batch-csv-prefix ".rels"))] (binding [*out* out] (let [rel-props (seq @rel-props)] (println (format "start\tend\t%1$s%2$s%3$s" (str/join "\t" rel-props) (if rel-props "\t" "") "l:label")) (doseq [rel (->> (keys @rels) sort)] (let [[start end] rel] (doseq [[labels props] (get @rels rel)] (println (format "%1$s\t%2$s\t%3$s%4$s%5$s" start end (str/join "\t" (map #(get props %) rel-props)) (if rel-props "\t" "") (str/join "," labels))))))))))) | |||||||||||||||||||||||||||||||||||||||
populate the database with the parsed apk structure | (defn populate-from-parsed-apk [apk {:keys [neo4j-include-methodinstance neo4j-no-callgraph] :as options}] (let [manifest (:manifest apk) dex-sha256 (:dex-sha256 apk) cert-sha256 (:cert-sha256 apk) apk-sha256 (:sha256 apk) apk-package (:package manifest) apk-version-name (:android:versionName manifest) apk-version-code (:android:versionCode manifest) dex (:dex apk) conn (connect options) transaction (ntx/begin-tx conn)] (ntx/with-transaction conn transaction true (ntx/execute conn transaction [(ntx/statement (str/join " " ["MERGE (signkey:SigningKey {sha256:{certsha256}})" "MERGE (apk:Apk {sha256:{apksha256},package:{apkpackage},versionCode:{apkversioncode},versionName:{apkversionname}})" "MERGE (dex:Dex {sha256:{dexsha256}})" "MERGE (signkey)-[:SIGN]->(apk)-[:CONTAIN]->(dex)" "FOREACH (" "perm in {usespermission} |" " MERGE (n:Permission {name:perm})" " MERGE (n)<-[:USE]-(apk)" ")" "FOREACH (" "perm in {permission} |" " MERGE (n:Permission {name:perm})" " MERGE (n)<-[:DEFINE]-(apk)" ")"]) {:certsha256 cert-sha256 :apksha256 apk-sha256 :dexsha256 dex-sha256 :usespermission (->> manifest :uses-permission (map name) ;; only consider Android internal API ones ;;(filter android-api?)) :permission (->> manifest :permission (map name) ;; only consider API ones ;;(filter android-api?)) :apkpackage apk-package :apkversionname apk-version-name :apkversioncode apk-version-code})]) (ntx/execute conn transaction (let [result (atom [])] (doseq [package-name (->> dex keys)] (let [class-names (->> (get-in dex [package-name]) keys)] (swap! result conj (ntx/statement (str/join " " ["MERGE (dex:Dex {sha256:{dexsha256}})" "MERGE (package:Package {name:{packagename}})" "FOREACH (" "classname in {classnames} |" " MERGE (class:Class {name:classname})" " MERGE (package)-[:CONTAIN]->(class)" " MERGE (dex)-[:CONTAIN]->(class)" ")"]) {:dexsha256 dex-sha256 :packagename package-name :classnames class-names})))) @result)) (ntx/execute conn transaction (let [result (atom [])] (doseq [package-name (->> dex keys)] (let [class-names (->> (get-in dex [package-name]) keys)] (doseq [class-name class-names] (let [{:keys [android-api-ancestors callbacks]} (->> (get-in dex [package-name class-name]))] (doseq [base android-api-ancestors] (let [ancestor-package (:package base) ancestor-class (:class base)] (swap! result conj (ntx/statement (str/join " " ["MERGE (class:Class {name:{classname}})" "MERGE (ancestorpackage:Package {name:{ancestorpackage}})" "MERGE (ancestorclass:Class {name:{ancestorclass}})" "MERGE (ancestorpackage)-[:CONTAIN]->(ancestorclass)" "MERGE (class)-[:DESCEND]->(ancestorclass)"]) {:classname class-name :ancestorpackage ancestor-package :ancestorclass ancestor-class})))))))) @result)) (ntx/execute conn transaction (let [result (atom [])] ;; http://stackoverflow.com/a/26366775 (dorun (for [package-name (->> dex keys)] (let [class-names (->> (get-in dex [package-name]) keys)] (dorun (for [class-name class-names] (let [{:keys [android-api-ancestors callbacks]} (->> (get-in dex [package-name class-name]))] (dorun (for [callback-name (->> callbacks keys)] (let [path [package-name class-name :callbacks callback-name]] (swap! result conj (ntx/statement (str/join " " ["MERGE (class:Class {name:{classname}})" "MERGE (callback:Method:Callback {name:{callbackname}})" "MERGE (class)-[:CONTAIN]->(callback)"]) {:classname class-name :callbackname (str class-name "." callback-name)})) ;; explicit control flow (let [path (conj path :explicit)] ;; deduplication (let [callback-invokes (->> (get-in dex path) (map #(select-keys % [:package :class :method])) (into #{}))] (dorun (for [callback-invoke callback-invokes] (let [invoke-package-name (:package callback-invoke) invoke-class-name (:class callback-invoke) invoke-name (:method callback-invoke)] (swap! result conj (ntx/statement (str/join " " ["MERGE (apk:Apk {sha256:{apksha256}})" "MERGE (callback:Callback {name:{callbackname}})" "MERGE (invokepackage:Package {name:{invokepackagename}})" "MERGE (invokeclass:Class {name:{invokeclassname}})" "MERGE (invoke:Method {name:{invokename}})" "MERGE (invokepackage)-[:CONTAIN]->(invokeclass)-[:CONTAIN]->(invoke)" "MERGE (callback)-[:EXPLICIT_INVOKE]->(invoke)" ;; to quickly find Apk from Method "MERGE (apk)<-[:INVOKED_BY]-(invoke)"]) {:apksha256 apk-sha256 :callbackname (str class-name "." callback-name) :invokepackagename invoke-package-name :invokeclassname invoke-class-name :invokename (str invoke-class-name "." invoke-name)})))))) (when neo4j-include-methodinstance (dorun (for [callback-invoke (get-in dex path)] (let [invoke-class-name (:class callback-invoke) invoke-name (:method callback-invoke) args (:args callback-invoke)] (swap! result conj (ntx/statement (str/join " " ["MERGE (callback:Callback {name:{callbackname}})" "MERGE (invoke:Method {name:{invokename}})" "MERGE (invokeinst:MethodInstance {name:{invokename},args:{args}})" "MERGE (invoke)<-[:INSTANCE_OF]-(invokeinst)" "MERGE (callback)-[:EXPLICIT_INVOKE]->(invokeinst)"]) {:callbackname (str class-name "." callback-name) :invokename (str invoke-class-name "." invoke-name) :args args}))))))) ;; implicit control flow (let [path (conj path :implicit)] ;; deduplication (let [callback-invokes (->> (get-in dex path) (map #(select-keys % [:package :class :method])) (into #{}))] (dorun (for [callback-invoke callback-invokes] (let [invoke-package-name (:package callback-invoke) invoke-class-name (:class callback-invoke) invoke-name (:method callback-invoke)] (swap! result conj (ntx/statement (str/join " " ["MERGE (apk:Apk {sha256:{apksha256}})" "MERGE (callback:Callback {name:{callbackname}})" "MERGE (invokepackage:Package {name:{invokepackagename}})" "MERGE (invokeclass:Class {name:{invokeclassname}})" "MERGE (invoke:Method {name:{invokename}})" "MERGE (invokepackage)-[:CONTAIN]->(invokeclass)-[:CONTAIN]->(invoke)" "MERGE (callback)-[:IMPLICIT_INVOKE]->(invoke)" ;; to quickly find Apk from Method "MERGE (apk)<-[:INVOKED_BY]-(invoke)"]) {:apksha256 apk-sha256 :callbackname (str class-name "." callback-name) :invokepackagename invoke-package-name :invokeclassname invoke-class-name :invokename (str invoke-class-name "." invoke-name)})))))) (when neo4j-include-methodinstance (dorun (for [callback-invoke (get-in dex path)] (let [invoke-class-name (:class callback-invoke) invoke-name (:method callback-invoke) args (:args callback-invoke)] (swap! result conj (ntx/statement (str/join " " ["MERGE (callback:Callback {name:{callbackname}})" "MERGE (invoke:Method {name:{invokename}})" "MERGE (invokeinst:MethodInstance {name:{invokename},args:{args}})" "MERGE (invoke)<-[:INSTANCE_OF]-(invokeinst)" "MERGE (callback)-[:IMPLICIT_INVOKE]->(invokeinst)"]) {:callbackname (str class-name "." callback-name) :invokename (str invoke-class-name "." invoke-name) :args args}))))))) ;; invokes that descend from Android API (let [path (conj path :descend)] (dorun (for [[api-invoke callback-invokes] (get-in dex path)] (let [api-package-name (:package api-invoke) api-class-name (:class api-invoke) api-name (:method api-invoke)] (swap! result conj (ntx/statement (str/join " " ["MERGE (apipackage:Package {name:{apipackagename}})" "MERGE (apiclass:Class {name:{apiclassname}})" "MERGE (apiname:Method {name:{apiname}})" "MERGE (apipackage)-[:CONTAIN]->(apiclass)-[:CONTAIN]->(apiname)"]) {:apipackagename api-package-name :apiclassname api-class-name :apiname (str api-class-name "." api-name)})) (dorun (for [callback-invoke callback-invokes] (let [invoke-package-name (:package callback-invoke) invoke-class-name (:class callback-invoke) invoke-name (:method callback-invoke)] (swap! result conj (ntx/statement (str/join " " ["MERGE (apiname:Method {name:{apiname}})" "MERGE (invokename:Method {name:{invokename}})" "MERGE (apiname)<-[:DESCEND]-(invokename)"]) (merge {:apiname (str api-class-name "." api-name) :invokename (str invoke-class-name "." invoke-name)})))))))))) (when-not neo4j-no-callgraph (let [path (conj path :invoke-paths) invoke-paths (get-in dex path)] (when-let [root-node (invoke-path-get-node invoke-paths)] ;; link the root node to the Callback and the Apk (swap! result conj (ntx/statement (str/join " " ["MERGE (apk:Apk {sha256:{apksha256}})" "MERGE (callback:Callback {name:{callbackname}})" "MERGE (cgnode:CallGraphNode {name:{name},apk:{apksha256},signature:{signature}})" "MERGE (apk)-[:CALLGRAPH]->(cgnode)<-[:CALLGRAPH]-(callback)"]) {:apksha256 apk-sha256 :name (invoke-path-get-node-name root-node) :signature root-node :callbackname (str class-name "." callback-name)}))) (process-worklist #{invoke-paths} (fn [worklist] (let [new-worklist (atom #{})] (dorun (for [work worklist] (let [node (invoke-path-get-node work) descendants (invoke-path-get-descendants work) children (map invoke-path-get-node descendants)] (swap! result conj (ntx/statement (str/join " " ["MERGE (node:CallGraphNode {name:{name},apk:{apksha256},signature:{signature}})" "FOREACH (" "child in {children} |" " MERGE (childnode:CallGraphNode {name:child.name,signature:child.signature,apk:{apksha256}})" " MERGE (node)-[:INVOKE]->(childnode)" ")"]) {:apksha256 apk-sha256 :name (invoke-path-get-node-name node) :signature node :children (map #(let [node %] {:name (invoke-path-get-node-name node) :signature node}) children)})) (swap! new-worklist into descendants)))) @new-worklist)))))))))))))) @result)) ;; app components (ntx/execute conn transaction (let [result (atom [])] (doseq [comp-type [:activity :service :receiver]] (doseq [[comp-name {:keys [intent-filter-action intent-filter-category]}] (->> manifest comp-type)] (let [comp-name (name comp-name) intent-filter-action (map name intent-filter-action) intent-filter-category (map name intent-filter-category)] (swap! result conj (ntx/statement (str/join " " ["MERGE (dex:Dex {sha256:{dexsha256}})" "MERGE (ic:Class {name:{compname}})" (format "SET ic:%1$s:Component" (->> comp-type name str/capitalize)) "MERGE (dex)-[:CONTAIN]->(ic)" "FOREACH (" "action IN {intentfilteraction} |" " MERGE (n:IntentFilterAction {name:action})" " MERGE (n)-[:TRIGGER]->(ic)" ")" "FOREACH (" "category IN {intentfiltercategory} |" " MERGE (n:IntentFilterCategory {name:category})" " MERGE (n)-[:TRIGGER]->(ic)" ")" ]) {:dexsha256 dex-sha256 :compname comp-name :intentfilteraction intent-filter-action :intentfiltercategory intent-filter-category}))))) @result)) ;; any more query within the transaction?))) | |||||||||||||||||||||||||||||||||||||||
tag an existing Apk node with the tagsuntag an existing Apk node with the tags | (let [common (fn [apk tags {:keys [verbose] :as options} op] (when-not (empty? tags) (let [statements (atom []) apk-sha256 (:sha256 apk)] (doseq [[types prop] tags] (swap! statements conj (ntx/statement (str/join " " ["MATCH (a:Apk {sha256:{apksha256}})" (format "MERGE (l:%1$s:Tag {id:{prop}.id})" (->> types ;; to satisfy Neo4j identifier requirement (map #(-> (str %) (str/replace #"\s+" "") (str/replace #"-" "_"))) (str/join ":"))) "SET l={prop}" "MERGE (l)-[r:TAG]->(a)" (case op :untag "DELETE r" :tag "" "")]) {:apksha256 apk-sha256 :prop prop}))) (let [conn (connect options) transaction (ntx/begin-tx conn)] (try (ntx/commit conn transaction @statements) (catch Exception e (print-stack-trace-if-verbose e verbose)))))))] (defn tag-apk [apk tags {:keys [] :as options}] (common apk tags options :tag)) (defn untag-apk [apk tags {:keys [] :as options}] (common apk tags options :untag))) | |||||||||||||||||||||||||||||||||||||||
add component callback signature | (defn add-callback-signature [apk {:keys [verbose] :as options}] (let [apk-sha256 (:sha256 apk) the-dex (:dex apk)] (dorun (for [comp-package-name (->> the-dex keys)] (do (dorun (for [comp-class-name (->> (get-in the-dex [comp-package-name]) keys)] (do (let [{:keys [android-api-ancestors callbacks]} (->> (get-in the-dex [comp-package-name comp-class-name]))] (dorun (for [callback-name (->> callbacks keys)] (do (let [path [comp-package-name comp-class-name :callbacks callback-name]] (let [path (conj path :invoke-paths) invoke-paths (get-in the-dex path)] (when invoke-paths (let [cgdfd (compute-cgdfd invoke-paths) signature (compute-cgdfd-signature cgdfd)] (when signature (let [statements (atom [])] (swap! statements conj (ntx/statement (str/join " " ["MATCH (a:Apk {sha256:{apksha256}})" "MATCH (cb:Callback {name:{callbackname}})" "MERGE (sig:CallbackSignature {name:{callbackname},apk:{apksha256}})" "MERGE (a)-[:CALLBACK_SIGNATURE]->(sig)<-[:CALLBACK_SIGNATURE]-(cb)" "SET sig.signature={signature}"]) {:apksha256 apk-sha256 :callbackname (str comp-class-name "." callback-name) :signature signature})) (let [conn (connect options) transaction (ntx/begin-tx conn)] (try (ntx/commit conn transaction @statements) (catch Exception e (print-stack-trace-if-verbose e verbose)))))))))))))))))))))) | |||||||||||||||||||||||||||||||||||||||
remove component callback signature | (defn remove-callback-signature [apk {:keys [verbose] :as options}] (let [apk-sha256 (:sha256 apk)] (let [statements (atom [])] (swap! statements conj (ntx/statement (str/join " " ["MATCH (a:Apk {sha256:{apksha256}})-[:CALLBACK_SIGNATURE]->(sig:CallbackSignature)" "WITH sig" "MATCH (sig)<-[e:CALLBACK_SIGNATURE]-()" "DELETE sig, e"]) {:apksha256 apk-sha256})) (let [conn (connect options) transaction (ntx/begin-tx conn)] (try (ntx/commit conn transaction @statements) (catch Exception e (print-stack-trace-if-verbose e verbose))))))) | |||||||||||||||||||||||||||||||||||||||
create index | (defn create-index [{:keys [] :as options}] (let [statements (map ntx/statement (map (fn [[label prop]] (str "CREATE INDEX ON :" label "(" prop ")")) [["SigningKey" "sha256"] ["Apk" "sha256"] ["Dex" "sha256"] ["Permission" "name"] ["Package" "name"] ["Class" "name"] ["Method" "name"] ["MethodInstance" "name"] ["Callback" "name"] ["Activity" "name"] ["Service" "name"] ["Receiver" "name"] ["IntentFilterAction" "name"] ["IntentFilterCategory" "name"] ["AndroidAPI" "name"] ["Tag" "id"] ["CallGraphNode" "apk"] ["CallGraphNode" "name"] ["CallbackSignature" "apk"] ["CallbackSignature" "name"]]))] (let [conn (connect options) transaction (ntx/begin-tx conn)] (ntx/commit conn transaction statements)))) | |||||||||||||||||||||||||||||||||||||||
label =~'^(?:com.)?android' nodes as AndroidAPI; should be infrequently used | (defn mark-android-api [{:keys [] :as options}] (let [conn (connect options) transaction (ntx/begin-tx conn)] (ntx/with-transaction conn transaction true (ntx/execute conn transaction [(ntx/statement (str/join " " ["MATCH (n)" "WHERE n.name=~{regex}" "SET n:AndroidAPI"]) {:regex "L?(?:android\\.|com\\.android\\.|dalvik\\.).*"})])))) | |||||||||||||||||||||||||||||||||||||||
connect to local neo4j server at PORT | (defn connect [{:keys [neo4j-port neo4j-protocol neo4j-conn-backoff verbose] :as options :or {neo4j-port (:neo4j-port @defaults) neo4j-protocol (:neo4j-protocol @defaults)}}] (let [port (if neo4j-port neo4j-port (:neo4j-port @defaults)) protocol (if neo4j-protocol neo4j-protocol (:neo4j-protocol @defaults)) retry (atom nil) conn (atom nil)] (loop [] (try (reset! conn (nr/connect (format "%1$s://localhost:%2$d/db/data/" protocol port))) (reset! retry false) ;; java.io.IOException catches java.net.SocketException and other situations (catch java.io.IOException e (let [backoff (rand-int neo4j-conn-backoff)] (when (and verbose (> verbose 1)) (binding [*out* *err*] (println "Neo4j connection exception, retry in" backoff "seconds"))) (Thread/sleep (* backoff 1000))) (reset! retry true))) (when @retry (recur))) @conn)) | |||||||||||||||||||||||||||||||||||||||
test whether NAME is part of Android API | (defn android-api? [name] (let [name (str name)] (re-find #"^L?(?:android\.|com\.android\.|dalvik\.)" name))) | |||||||||||||||||||||||||||||||||||||||
recipes for queries | (ns woa.neo4j.recipe ;; common libs (:require [clojure.string :as str] [clojure.set :as set] [clojure.walk :as walk] [clojure.zip :as zip] [clojure.java.io :as io] [clojure.pprint :refer [pprint print-table]] [clojure.stacktrace :refer [print-stack-trace]])) | |||||||||||||||||||||||||||||||||||||||
declaration | ||||||||||||||||||||||||||||||||||||||||
(declare get-app-skeleton get-app-by-class-complexity) | ||||||||||||||||||||||||||||||||||||||||
implementation | ||||||||||||||||||||||||||||||||||||||||
get the skeleton of an app | (defn get-app-skeleton [{:keys [sha256 package versionCode versionName] :as apk} & {:keys [return?] :as options}] (str/join " " [(format "MATCH (apk:Apk%1$s)" (if apk (format "{%1$s}" (str/join "," (->> apk (map (fn [[k v]] (str (name k) ":" (format "\"%1$s\"" v))))))) "")) "OPTIONAL MATCH (signingKey:SigningKey)" "-[:SIGN]-> (apk)" "-[:CONTAIN]-> (dex:Dex)" "-[:CONTAIN]-> (class:Class)" "-[:CONTAIN]-> (callback:Callback)" "OPTIONAL MATCH (callback) -[:INVOKE]-> (invoke)" "OPTIONAL MATCH (explicitInvoke) <-[:EXPLICIT_INVOKE]- (callback) -[:IMPLICIT_INVOKE]-> (implicitInvoke)" (if return? "RETURN signingKey, apk, dex, class, callback, invoke, explicitInvoke, implicitInvoke" "")])) | |||||||||||||||||||||||||||||||||||||||
sort apps by how many component classes they have | (defn get-app-by-class-complexity [& {:keys [skip limit desc where return?] :as options :or {limit 5}}] (str/join " " ["MATCH (apk:Apk)" "-[:CONTAIN]-> (:Dex)" "-[:CONTAIN]-> (class:Class)" "WITH apk, count(class) as cc" (if where (format "WHERE %1$s" where) "") (if return? "RETURN apk, cc" "") "ORDER BY cc" (if desc "DESC" "") (if limit (format "LIMIT %1$d" limit) "") (if skip (format "SKIP %1$d" skip) "") ])) | |||||||||||||||||||||||||||||||||||||||
(ns woa.util ;; common libs (:require [clojure.string :as str] [clojure.set :as set] [clojure.walk :as walk] [clojure.zip :as zip] [clojure.java.io :as io] [clojure.pprint :refer [pprint print-table]] [clojure.stacktrace :refer [print-stack-trace]]) ;; imports) | ||||||||||||||||||||||||||||||||||||||||
declaration | ||||||||||||||||||||||||||||||||||||||||
(declare process-worklist print-stack-trace-if-verbose) | ||||||||||||||||||||||||||||||||||||||||
implementation | ||||||||||||||||||||||||||||||||||||||||
process worklist until it is empty process takes a worklist as input, and outputs the new worklist | (defn process-worklist [initial-worklist process] (loop [worklist initial-worklist] (when-not (empty? worklist) (recur (process worklist))))) | |||||||||||||||||||||||||||||||||||||||
print-stack-trace Exception e to err if verbose is non-nil | (defn print-stack-trace-if-verbose [^Exception e verbose & [level]] (when (and verbose (or (not level) (> verbose level))) (binding [*out* *err*] (print-stack-trace e) ;; flush is critical for timely output (flush)))) | |||||||||||||||||||||||||||||||||||||||
https://www.virustotal.com/en/documentation/public-api/ | (ns woa.virustotal.core ;; internal libs ;; common libs (:require [clojure.string :as str] [clojure.set :as set] [clojure.walk :as walk] [clojure.zip :as zip] [clojure.java.io :as io] [clojure.pprint :refer [pprint print-table]] [clojure.stacktrace :refer [print-stack-trace]]) ;; special libs (:require [clj-http.client :as http-client] [clojure.data.json :as json])) | |||||||||||||||||||||||||||||||||||||||
declaration | ||||||||||||||||||||||||||||||||||||||||
(declare submit-file request-rescan make-report-result-into-tags get-report ;; plumbing jsonfy-map clojurefy-map interpret-response-code interpret-status) | ||||||||||||||||||||||||||||||||||||||||
implementation | ||||||||||||||||||||||||||||||||||||||||
porcelain | ||||||||||||||||||||||||||||||||||||||||
submit file for checking | (defn submit-file [{:keys [file-content] :as resource} {:keys [virustotal-apikey] :as options}] (let [url "https://www.virustotal.com/vtapi/v2/file/scan"] ;; basic safety check (when (and file-content) (let [{:keys [status body] :as http-response} (http-client/post url {:multipart [{:name "apikey" :content virustotal-apikey} {:name "file" :content file-content}] :throw-exceptions false}) i-status (interpret-status status)] (if (= i-status :status-ok) (let [result (->> body json/read-str clojurefy-map) {:keys [response-code]} result i-response-code (interpret-response-code response-code)] (if (= i-response-code :response-ok) (assoc-in result [:response-code] i-response-code) i-response-code)) i-status))))) | |||||||||||||||||||||||||||||||||||||||
obtain report result | (defn request-rescan [{:keys [md5 sha1 sha256 scan-id] :as resource} {:keys [virustotal-apikey] :as options}] (let [url "https://www.virustotal.com/vtapi/v2/file/rescan"] (when-let [resource (or md5 sha1 sha256 scan-id)] (let [{:keys [status body] :as http-response} (http-client/post url {:form-params {:apikey virustotal-apikey :resource resource} :throw-exceptions false}) i-status (interpret-status status)] (if (= i-status :status-ok) (let [result (->> body json/read-str clojurefy-map) {:keys [response-code]} result i-response-code (interpret-response-code response-code)] (if (= i-response-code :response-ok) (assoc-in result [:response-code] i-response-code) i-response-code)) i-status))))) | |||||||||||||||||||||||||||||||||||||||
make report-result into the form suitable to supply as app :tags on command-line | (defn make-report-result-into-tags [report-result] (when-let [scans (:scans report-result)] (->> scans (filter (fn [[_ v]] (get v "detected"))) (map (fn [[k v]] [["Malware" "VirusTotal"] (let [result (get v "result")] (merge (dissoc v "detected") {"id" (str/join "-" ["malware" k result]) "source" k}))])) vec))) | |||||||||||||||||||||||||||||||||||||||
obtain report result | (defn get-report [{:keys [md5 sha1 sha256 scan-id] :as resource} {:keys [virustotal-apikey] :as options}] (let [url "https://www.virustotal.com/vtapi/v2/file/report"] (when-let [resource (or md5 sha1 sha256 scan-id)] (let [{:keys [status body] :as http-response} (http-client/post url {:form-params {:apikey virustotal-apikey :resource resource} :throw-exceptions false}) i-status (interpret-status status)] (if (= i-status :status-ok) (let [result (->> body json/read-str clojurefy-map) {:keys [response-code]} result i-response-code (interpret-response-code response-code)] (if (= i-response-code :response-ok) (assoc-in result [:response-code] i-response-code) i-response-code)) i-status))))) | |||||||||||||||||||||||||||||||||||||||
plumbing | ||||||||||||||||||||||||||||||||||||||||
interpret HTTP response code from VirusTotal | (defn interpret-response-code [response-code] (case response-code 0 :response-not-found 1 :response-ok -2 :response-still-queued (list :response-code response-code))) | |||||||||||||||||||||||||||||||||||||||
interpret HTTP status from VirusTotal | (defn interpret-status [status] (case status 200 :status-ok 204 :status-exceed-api-limit 403 :status-exceed-priviledge (list :status status))) | |||||||||||||||||||||||||||||||||||||||
use underscored string as key | (defn jsonfy-map [the-map] (->> the-map (map (fn [[k v]] [(-> (cond (keyword? k) (name k) :otherwise (str k)) (str/replace "-" "_")) v])) (into {}))) | |||||||||||||||||||||||||||||||||||||||
use dashed keyword as key | (defn clojurefy-map [the-map] (->> the-map (map (fn [[k v]] [(-> k (str/replace "_" "-") keyword) v])) (into {}))) | |||||||||||||||||||||||||||||||||||||||