diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..7c62f427e903738d520df0577546ab2c94c6f861 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +ARG BASE_REGISTRY=registry1.dso.mil +ARG BASE_IMAGE=ironbank/opensource/palo-alto-networks/demisto/python +ARG BASE_TAG=2.7.18.20958 +FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG} + +COPY requirements.txt . + +RUN mkdir ./pip-pkgs + +COPY *.* ./pip-pkgs/ + +USER root + +RUN dnf install -y --nodocs python2-devel gcc gcc-c++ make wget git && \ + pip install --no-cache-dir --no-index --find-links ./pip-pkgs/ -r requirements.txt && \ + dnf remove -y python2-devel gcc gcc-c++ make wget git && \ + dnf clean all && \ + rm -rf /var/cache/dnf && \ + rm -rf ./pip-pkgs + +ENV DOCKER_IMAGE='demisto/dempcap:1.0.0.23135' + +ENV DOCKER_IMAGE_IRONBANK='opensource/palo-alto-networks/demisto/dempcap:1.0.0.23135' + +# Copy the pcapminey directory contents into the container at /app +ADD pcapminey /app/pcapminey + +HEALTHCHECK NONE + diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..d9cada66c91e39f989e3ce7ee6c1bddfb5b305ed --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2018 Demisto + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index 5dc6fa6db4361c22da2f35edf0544d83ba6001e2..155d67d60c7dbe320d6891de767bbc264ef61bc0 100644 --- a/README.md +++ b/README.md @@ -1,3 +1 @@ -# - -Project template for all Iron Bank container repositories. \ No newline at end of file +Palo Alto Networks - Demisto XSOAR - dempcap image with the required dependencies diff --git a/hardening_manifest.yaml b/hardening_manifest.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ef20203bad3dfa1e3e944a191bbf9b96350891f4 --- /dev/null +++ b/hardening_manifest.yaml @@ -0,0 +1,43 @@ +apiVersion: v1 +name: opensource/palo-alto-networks/demisto/dempcap +tags: +- 1.0.0.23980 +args: + BASE_IMAGE: opensource/palo-alto-networks/demisto/python + BASE_TAG: 2.7.18.20958 +labels: + org.opencontainers.image.title: Demisto Automation - dempcap image + org.opencontainers.image.description: dempcap image with the required dependencies + org.opencontainers.image.licenses: ' ' + org.opencontainers.image.url: ' ' + org.opencontainers.image.vendor: demisto + org.opencontainers.image.version: '1.0' + mil.dso.ironbank.image.keywords: cymruwhois, dpkt, regex, simplejson + mil.dso.ironbank.image.type: opensource + mil.dso.ironbank.product.name: panw-demisto-dempcap +resources: +- filename: cymruwhois-1.6-py2-none-any.whl + url: https://files.pythonhosted.org/packages/ef/65/f2b9f0fdf0cde2ce9ee68256bcbbf8ad8a05def3cc913a53e498b165fe97/cymruwhois-1.6-py2-none-any.whl + validation: + type: sha256 + value: 0294accc8b0668abf5b33b6b5151fc3b1ff4d02f27549ffa258945ea9e53ccad +- filename: dpkt-1.9.4.tar.gz + url: https://files.pythonhosted.org/packages/aa/47/0ba0e71b750ed3907c21f0d3ce50de89a066e18c0bfeb13f99a552a524b4/dpkt-1.9.4.tar.gz + validation: + type: sha256 + value: f4e579cbaf6e2285ebf3a9e84019459b4367636bac079ba169527e582fca48b4 +- filename: regex-2020.11.13.tar.gz + url: https://files.pythonhosted.org/packages/2e/e4/3447fed9ab29944333f48730ecff4dca92f0868c5b188d6ab2b2078e32c2/regex-2020.11.13.tar.gz + validation: + type: sha256 + value: 83d6b356e116ca119db8e7c6fc2983289d87b27b3fac238cfe5dca529d884562 +- filename: simplejson-3.17.2-cp27-cp27mu-manylinux2010_x86_64.whl + url: https://files.pythonhosted.org/packages/90/68/bced6ede6012c9c25a555e149238e16b85bc091940d605f16bd3b499d16e/simplejson-3.17.2-cp27-cp27mu-manylinux2010_x86_64.whl + validation: + type: sha256 + value: c1cb29b1fced01f97e6d5631c3edc2dadb424d1f4421dad079cb13fc97acb42f +maintainers: +- email: containers@demisto.com + name: Palo Alto Networks + username: gfreund + cht_member: false diff --git a/pcapminey/.gitignore b/pcapminey/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..309f5ed4894fcb0642d14393cb8e0cbd844b2531 --- /dev/null +++ b/pcapminey/.gitignore @@ -0,0 +1 @@ +# Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] # C extensions *.so # Distribution / packaging .Python env/ build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ *.egg-info/ .installed.cfg *.egg # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *,cover # Translations *.mo *.pot # Django stuff: *.log # Sphinx documentation docs/_build/ # PyBuilder target/ # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm *.iml ## Directory-based project format: .idea/ # if you remove the above rule, at least ignore the following: # User-specific stuff: # .idea/workspace.xml # .idea/tasks.xml # .idea/dictionaries # .idea/shelf # Sensitive or high-churn files: # .idea/dataSources.ids # .idea/dataSources.xml # .idea/sqlDataSources.xml # .idea/dynamic.xml # .idea/uiDesigner.xml # Gradle: # .idea/gradle.xml # .idea/libraries # Mongo Explorer plugin: # .idea/mongoSettings.xml ## File-based project format: *.ipr *.iws ## Plugin-specific files: # IntelliJ /out/ # mpeltonen/sbt-idea plugin .idea_modules/ # JIRA plugin atlassian-ide-plugin.xml # Crashlytics plugin (for Android Studio and IntelliJ) com_crashlytics_export_strings.xml crashlytics.properties crashlytics-build.properties fabric.properties \ No newline at end of file diff --git a/pcapminey/LICENSE b/pcapminey/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..8f71f43fee3f78649d238238cbde51e6d7055c82 --- /dev/null +++ b/pcapminey/LICENSE @@ -0,0 +1,202 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/pcapminey/META-INF/MANIFEST.MF b/pcapminey/META-INF/MANIFEST.MF new file mode 100644 index 0000000000000000000000000000000000000000..b46e523ab9dd7d279bb44e61ded85a1ef9903b09 --- /dev/null +++ b/pcapminey/META-INF/MANIFEST.MF @@ -0,0 +1,3 @@ +Manifest-Version: 1.0 +Created-By: 1.7.0_10 (Oracle Corporation) + diff --git a/pcapminey/README.md b/pcapminey/README.md new file mode 100644 index 0000000000000000000000000000000000000000..992c264828006b5ce6d8681c88b956c20658c5af --- /dev/null +++ b/pcapminey/README.md @@ -0,0 +1,51 @@ +# pcapfex +'**P**acket **CAP**ture **F**orensic **E**vidence e**X**tractor' is a tool +that finds and extracts files from packet capture files. + +It was developed by _Viktor Winkelmann_ as part of a bachelor thesis. + +The power of _pcapfex_ lies in it's ease of use. You only provide it a +pcap-file and are rewarded a structured export of all files found in it. +_pcacpfex_ allows data extraction even if non-standard protocols were used. +It's easy to understand plugin-system offers python developers a quick way +to add more file-types, encodings or +even complex protocols. + +### Requirements +_pcapfex_ was developed and tested for **Linux environments only**. +Due to missing optimizations and tests, there is no guarantee for it to work +under Windows (though it should work). + +_pcapfex_ depends on **Python 2.7** and the **_dpkt_** package. You can install +it via +``` +sudo pip install dpkt +``` + +To achieve better performance using a multithreaded search for file objects, you +should install the _regex_ package. +``` +sudo pip install regex +``` +However, this step is only optional. + + +### Usage +To analyze a pcap-file ```samplefile.pcap``` just use +``` +pcapfex.py samplefile.pcap +``` + + +For more detailed usage information see +``` +pcapfex.py -h +``` + +Please make sure to use the ```-nv``` flag, if the machine +that captured the traffic was sending data as well. This will +circumvent wrong checksums stored in the pcap-file caused by +TCP-Checksum-Offloading. + +### License +_pcapfex_ is published under the Apache 2.0 license. \ No newline at end of file diff --git a/pcapminey/__init__.py b/pcapminey/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..61cc9dc341e9a306e4af0d7d2f02b18b86f76eca --- /dev/null +++ b/pcapminey/__init__.py @@ -0,0 +1 @@ +__author__ = 'Viktor Winkelmann' diff --git a/pcapminey/core/Dispatcher.py b/pcapminey/core/Dispatcher.py new file mode 100644 index 0000000000000000000000000000000000000000..b65ec594eb7aa2fbddaa8dade505d92dac632241 --- /dev/null +++ b/pcapminey/core/Dispatcher.py @@ -0,0 +1,103 @@ +# -*- coding: utf8 -*- +__author__ = 'Viktor Winkelmann' + +import multiprocessing +from ThreadPool.Pool import Pool +from Files.FileManager import * +from Files.FileObject import * +from Streams.StreamBuilder import * +from Plugins.PluginManager import * + +DEBUG = False + +class Dispatcher: + def __init__(self, pcapfile, outputdir='output', entropy=False, **kwargs): + self.kwargs = kwargs + self.pcapfile = pcapfile + self.filemanager = FileManager(outputdir) + self.pm = PluginManager() + self.outputdir = outputdir + self.useEntropy = entropy + + def _finishedSearch(self, (stream, result)): + #Utils.printl("Found %d files in %s stream %s" % (len(result), stream.protocol, stream.infos)) + map(self.filemanager.addFile, result) + + def run(self): + #if os.path.exists(self.outputdir): + #print "Output folder \'%s\' already exists! Exiting..." % (self.outputdir,) + #self.filemanager.exit() + #return + + + #print "Reassembling streams..." + streambuilder = StreamBuilder(self.pcapfile, **self.kwargs) + allstreams = streambuilder.tcpStreams + streambuilder.udpStreams + + #print "\nStream Reassembly finished.\n\tFile %s has a total of %d single-direction streams." % (self.pcapfile,len(allstreams)) + + #print "Searching streams for forensic evidence...\n" + if DEBUG: + # Single threaded search for easier debugging + map(lambda s: self._finishedSearch(self._findFiles(s)), allstreams) + else: + # Multi threaded search as standard + workers = Pool(multiprocessing.cpu_count()) + workers.map_async(self._findFiles, allstreams, self._finishedSearch) + workers.join() + + + + self.filemanager.exit() + #print "Evidence search has finished.\n" + return self.filemanager + + + + def _findFiles(self, stream): + files = [] + payloads= [] + streamdata = stream.getAllBytes() + streamPorts = (stream.ipSrc, stream.ipDst) + + for protocol in self.pm.getProtocolsByHeuristics(streamPorts): + payloads = self.pm.protocolDissectors[protocol].parseData(streamdata) + + if payloads is not None: + stream.protocol = self.pm.protocolDissectors[protocol].protocolName + break + + for encPayload in payloads: + for decoder in self.pm.decoders: + payload = self.pm.decoders[decoder].decodeData(encPayload) + if payload is None: + continue + + + for datarecognizer in self.pm.dataRecognizers: + for occ in self.pm.dataRecognizers[datarecognizer].findAllOccurences(payload): + file = FileObject(payload[occ[0]:occ[1]]) + file.source = stream.ipSrc + file.destination = stream.ipDst + file.fileEnding = self.pm.dataRecognizers[datarecognizer].fileEnding + file.type = self.pm.dataRecognizers[datarecognizer].dataCategory + if stream.tsFirstPacket: + file.timestamp = stream.tsFirstPacket + files.append(file) + + if self.useEntropy: + type = self.pm.entropyClassifier.classify(payload) + file = FileObject(payload) + file.source = stream.ipSrc + file.destination = stream.ipDst + file.type = type + if stream.tsFirstPacket: + file.timestamp = stream.tsFirstPacket + files.append(file) + + + return (stream, files) + +if __name__ == '__main__': + d = Dispatcher(os.path.dirname(__file__) + '/../tests/webextract/web_light.pcap') + d.run() diff --git a/pcapminey/core/Files/FileManager.py b/pcapminey/core/Files/FileManager.py new file mode 100644 index 0000000000000000000000000000000000000000..5363ee9c96090ac273312dd358d7b599aaf43f08 --- /dev/null +++ b/pcapminey/core/Files/FileManager.py @@ -0,0 +1,62 @@ +# -*- coding: utf8 -*- +__author__ = 'Viktor Winkelmann' + +import sys +sys.path.append('../..') + +import os +import core.Utils as Utils +from Queue import Queue +from threading import Thread + +class FileManager(Thread): + def __init__(self, outputdir): + Thread.__init__(self) + self.setName('FileManager thread') + self.daemon = True + + self.files = Queue() + self.outputdir = outputdir + self.filenamelist = [] + self.stop = False + self.start() + + def addFile(self, file): + self.files.put(file) + + def exit(self): + self.files.join() + self.stop = True + self.files.put(None) + self.join() + + def run(self): + while not self.stop or not self.files.empty(): + file = self.files.get() + + if not file: + self.files.task_done() + continue + + newtimestamp = file.timestamp.replace(" ", "_") + filenamepath = "%s_%ss_%s_%s_%s_" % (self.outputdir, file.type, file.source, file.destination, newtimestamp) + path = "%s/%ss/%s_%s/%s/" % (self.outputdir, file.type, file.source, file.destination, newtimestamp) + path2 = "%s/" % (self.outputdir) + if not os.path.exists(path2): + os.makedirs(path2) + + number = 1 + filename = '%s_%d.%s' % (file.name, number, file.fileEnding) + #while os.path.exists(path + filename): + while os.path.exists(path2 + filename): + number += 1 + filename = '%s_%d.%s' % (file.name, number, file.fileEnding) + + filename = filenamepath + filename.rstrip('.') + #with open(path + filename, 'wb') as outfile: + with open(path2+filename, 'wb') as outfile: + outfile.write(file.data) + self.filenamelist.append(filename) + #Utils.printl("Wrote file: %s%s" % (path2, filename)) + + self.files.task_done() diff --git a/pcapminey/core/Files/FileObject.py b/pcapminey/core/Files/FileObject.py new file mode 100644 index 0000000000000000000000000000000000000000..19c85402b0781f1b11d39fe04fe965d8a6c6dbc9 --- /dev/null +++ b/pcapminey/core/Files/FileObject.py @@ -0,0 +1,37 @@ +# -*- coding: utf8 -*- +__author__ = 'Viktor Winkelmann' + +import datetime + +class FileObject(object): + def __init__(self, data): + self.data = data + self._name = None + self.source = 'unknown' + self.destination = 'unknown' + self._timestamp = 'unknown' + self.type = 'unknown' + self.fileEnding = 'unknown' + + @property + def name(self): + if self._name: + return self._name + else: + return self.type.split('/')[-1] + + @name.setter + def name(self, value): + self._name = value + + + @property + def timestamp(self): + return self._timestamp + + @timestamp.setter + def timestamp(self, value): + try: + self._timestamp = str(datetime.datetime.utcfromtimestamp(value)).replace(':', '-') + except ValueError: + self._timestamp = value \ No newline at end of file diff --git a/pcapminey/core/Files/__init__.py b/pcapminey/core/Files/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f79193bf8b2cb2540c7a17dfe31f1945fa5cbdbe --- /dev/null +++ b/pcapminey/core/Files/__init__.py @@ -0,0 +1,2 @@ +__author__ = 'Viktor Winkelmann' +__all__ = ['FileManager', 'FileObject'] \ No newline at end of file diff --git a/pcapminey/core/Plugins/DataRecognizer.py b/pcapminey/core/Plugins/DataRecognizer.py new file mode 100644 index 0000000000000000000000000000000000000000..49da5773a1d3b3222cfafd11cb18f3064df12903 --- /dev/null +++ b/pcapminey/core/Plugins/DataRecognizer.py @@ -0,0 +1,94 @@ +# -*- coding: utf8 -*- +__author__ = 'Viktor Winkelmann' + +from abc import ABCMeta, abstractproperty +from Plugin import * +try: + import regex as re + #print 'Using concurrency enabled regex module.' +except: + #print 'Consider installing the \'regex\' module using \'pip install regex\' to improve performance on multicore systems.' + import re + +#from pympler import tracker + +class DataCategory: + IMAGE = "Image_file" + VIDEO = "Video_file" + AUDIO = "Audio_file" + DOC = "Document_file" + TEXT = "Plaintext_file" + EXECUTABLE = "Executable_file" + + COMPRESSED = "Compressed_file" + ENCRYPTED = "Encrypted_file" + UNKNOWN = "Unknown_data" + + def __iter__(self): + return self.__dict__.__iter__() + + +class DataRecognizer(Plugin): + __metaclass__ = ABCMeta + + @classmethod + def getPriority(cls): + return cls.basePriority + + + @abstractproperty + def signatures(cls): + """ IMPORTANT: Override as Class Property """ + return NotImplemented + + @abstractproperty + def fileEnding(cls): + """ IMPORTANT: Override as Class Property """ + return NotImplemented + + @abstractproperty + def dataType(cls): + """ IMPORTANT: Override as Class Property """ + return NotImplemented + + @abstractproperty + def dataCategory(cls): + """ IMPORTANT: Override as Class Property """ + return NotImplemented + + @classmethod + def _buildRegexPatterns(cls): + regexstr = b'' + for (fileHeader, fileTrailer) in cls.signatures: + if fileTrailer is None: + regexstr += b'(%s.*)|' % (fileHeader,) + else: + regexstr += b'(%s.*?%s)|' % (fileHeader, fileTrailer) + + cls._regex = re.compile(regexstr[:-1], re.DOTALL) + + @classmethod + def findNextOccurence(cls, data, startindex=0, endindex=0): + if not hasattr(cls, '_regex'): + cls._buildRegexPatterns() + + if endindex == 0: + endindex = len(data) + + match = cls._regex.search(data, startindex, endindex) + + return match.span() if match else None + + @classmethod + def findAllOccurences(cls, data, startindex=0, endindex=0): + if not hasattr(cls, '_regex'): + cls._buildRegexPatterns() + + if endindex == 0: + endindex = len(data) + + #tr = tracker.SummaryTracker() + occurences = [m.span() for m in cls._regex.finditer(data, startindex, endindex)] + + #tr.print_diff() + return occurences diff --git a/pcapminey/core/Plugins/Decoder.py b/pcapminey/core/Plugins/Decoder.py new file mode 100644 index 0000000000000000000000000000000000000000..75e775093f8765dd97cda48affa59c4b86a1dc4f --- /dev/null +++ b/pcapminey/core/Plugins/Decoder.py @@ -0,0 +1,22 @@ +# -*- coding: utf8 -*- +__author__ = 'Viktor Winkelmann' + +from abc import ABCMeta, abstractmethod, abstractproperty +from Plugin import * + +class Decoder(Plugin): + __metaclass__ = ABCMeta + + @abstractproperty + def decoderName(cls): + """ IMPORTANT: Override as Class Property """ + return NotImplemented + + @classmethod + def getPriority(cls): + return cls.basePriority + + @abstractmethod + def decodeData(cls, data): + """ IMPORTANT: Override as Class Method (using @classmethod) """ + return NotImplemented diff --git a/pcapminey/core/Plugins/EntropyClassifier.py b/pcapminey/core/Plugins/EntropyClassifier.py new file mode 100644 index 0000000000000000000000000000000000000000..80f36ad6942ac269dcf4299f0d4b9e0a339c8ad3 --- /dev/null +++ b/pcapminey/core/Plugins/EntropyClassifier.py @@ -0,0 +1,54 @@ +# -*- coding: utf8 -*- +__author__ = 'Viktor Winkelmann' +import math + +class EntropyClass: + PLAIN = 'rawdata_plain_file' + COMPRESSED = 'rawdata_compressed_file' + ENCRYPTED = 'rawdata_encrypted_file' + UNCATEGORIZED = 'rawdata_uncategorized_file' + +class EntropyClassifier: + RANDOM_ENTROPY = 7.9 + CHI2_LOWER_BOUND = 206 + CHI2_UPPER_BOUND = 311 + + MINIMUM_DATA_LENGTH = 256 * 5 + MAXIMUM_DATA_LENGTH = 1024 * 100 + + @classmethod + def classify(cls, data): + data = data[:cls.MAXIMUM_DATA_LENGTH] + l = len(data) + if l < cls.MINIMUM_DATA_LENGTH: + return EntropyClass.UNCATEGORIZED + + h = cls._histogram(data) + s = cls._shannonEntropy(h, l) + + if s >= cls.RANDOM_ENTROPY: + c = cls._chiSquare(h, l) + if c > cls.CHI2_LOWER_BOUND and c < cls.CHI2_UPPER_BOUND: + return EntropyClass.ENCRYPTED + else: + return EntropyClass.COMPRESSED + + else: + return EntropyClass.PLAIN + + @classmethod + def _shannonEntropy(cls, histogram, datalength): + return abs(reduce(lambda acc,x: acc + (x/datalength) * math.log(x/datalength, 2) if x > 0 else acc + x, + histogram, 0)) + + @classmethod + def _chiSquare(cls, histogram, datalength): + exp = float(datalength) / 256 + return reduce(lambda acc,x: acc + ((x-exp)**2)/exp, histogram, 0) + + @classmethod + def _histogram(cls, data): + h = [0.0] * 256 + for char in data: + h[ord(char)] += 1.0 + return h diff --git a/pcapminey/core/Plugins/Plugin.py b/pcapminey/core/Plugins/Plugin.py new file mode 100644 index 0000000000000000000000000000000000000000..ea88f8be3d8f9422ae8db35e96de9eb67ff31c58 --- /dev/null +++ b/pcapminey/core/Plugins/Plugin.py @@ -0,0 +1,14 @@ +# -*- coding: utf8 -*- +__author__ = 'Viktor Winkelmann' + +from abc import ABCMeta, abstractmethod + +class Plugin: + __metaclass__ = ABCMeta + + basePriority = 50 + + @abstractmethod + def getPriority(cls): + """ IMPORTANT: Override as Class Method (using @classmethod) """ + return NotImplemented \ No newline at end of file diff --git a/pcapminey/core/Plugins/PluginManager.py b/pcapminey/core/Plugins/PluginManager.py new file mode 100644 index 0000000000000000000000000000000000000000..8fb0654a4d9302550c87a0496ee0ddfd2ea4065c --- /dev/null +++ b/pcapminey/core/Plugins/PluginManager.py @@ -0,0 +1,51 @@ +# -*- coding: utf8 -*- +__author__ = 'Viktor Winkelmann' +import os, imp +from collections import OrderedDict +from EntropyClassifier import EntropyClassifier + + +class PluginManager: + PD_PATH = '/../../plugins/protocol_dissectors/' + DR_PATH = '/../../plugins/data_recognizers/' + DC_PATH = '/../../plugins/decoders/' + + def __init__(self): + modulepath = os.path.dirname(__file__) + self.protocolDissectors = OrderedDict() + self.dataRecognizers = OrderedDict() + self.decoders = OrderedDict() + self.entropyClassifier = EntropyClassifier() + self.__loadPlugins(modulepath + self.__class__.PD_PATH, self.protocolDissectors) + self.__loadPlugins(modulepath + self.__class__.DR_PATH, self.dataRecognizers) + self.__loadPlugins(modulepath + self.__class__.DC_PATH, self.decoders) + + self.protocolDissectors = OrderedDict( + sorted(self.protocolDissectors.iteritems(), key=lambda x: x[1].getPriority())) + self.dataRecognizers = OrderedDict( + sorted(self.dataRecognizers.iteritems(), key=lambda x: x[1].getPriority())) + self.decoders = OrderedDict( + sorted(self.decoders.iteritems(), key=lambda x: x[1].getPriority())) + + + def getProtocolsByHeuristics(self, streamPorts): + return OrderedDict( + sorted(self.protocolDissectors.iteritems(), key=lambda x: x[1].getPriority(streamPorts))) + + def __loadPlugins(self, path, targetdict): + for pluginfile in os.listdir(path): + if not os.path.isfile(path + pluginfile): + continue + if not pluginfile[-3:] == ".py": + continue + if pluginfile.endswith("__init__.py"): + continue + + name = pluginfile.split('/')[-1][:-3] + module = imp.load_source(name, path + pluginfile) + targetdict[name] = module.getClassReference() + + +if __name__ == "__main__": + pm = PluginManager() + #print pm.protocolDissectors['http11'].getPriority() diff --git a/pcapminey/core/Plugins/ProtocolDissector.py b/pcapminey/core/Plugins/ProtocolDissector.py new file mode 100644 index 0000000000000000000000000000000000000000..41a9beb1eae95b1ffcc46621f0da3af880bd5113 --- /dev/null +++ b/pcapminey/core/Plugins/ProtocolDissector.py @@ -0,0 +1,26 @@ +# -*- coding: utf8 -*- +__author__ = 'Viktor Winkelmann' + +from abc import ABCMeta, abstractmethod, abstractproperty +from Plugin import * + +class ProtocolDissector(Plugin): + __metaclass__ = ABCMeta + defaultPorts = [] + + @abstractproperty + def protocolName(cls): + """ IMPORTANT: Override as Class Property """ + return NotImplemented + + @classmethod + def getPriority(cls, streamPorts = ()): + if any(map(lambda x: x in cls.defaultPorts, streamPorts)): + return cls.basePriority - 50 + + return cls.basePriority + + @abstractmethod + def parseData(cls, data): + """ IMPORTANT: Override as Class Method (using @classmethod) """ + return NotImplemented diff --git a/pcapminey/core/Plugins/__init__.py b/pcapminey/core/Plugins/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..cb7ccbee4cc8878c10925a52b389a0104813a938 --- /dev/null +++ b/pcapminey/core/Plugins/__init__.py @@ -0,0 +1,2 @@ +__author__ = 'Viktor Winkelmann' +__all__ = ['DataRecognizer', 'PluginManager', 'ProtocolDissector'] \ No newline at end of file diff --git a/pcapminey/core/Streams/PacketStream.py b/pcapminey/core/Streams/PacketStream.py new file mode 100644 index 0000000000000000000000000000000000000000..02ac9f30ac04e461a6e34bedbd9d3051cd44e544 --- /dev/null +++ b/pcapminey/core/Streams/PacketStream.py @@ -0,0 +1,46 @@ +# -*- coding: utf8 -*- +__author__ = 'Viktor Winkelmann' +from abc import ABCMeta, abstractmethod + + +class PacketStream: + __metaclass__ = ABCMeta + + def __init__(self, ipSrc, portSrc, ipDst, portDst): + self.ipSrc = ipSrc + self.portSrc = portSrc + self.ipDst = ipDst + self.portDst = portDst + self.infos = "%s:%s to %s:%s" % (ipSrc, portSrc, ipDst, portDst) + self.protocol = 'unknown protocol' + self.tsFirstPacket = None + self.tsLastPacket = None + self.closed = False + + # def __eq__(self, other): + # if other is None: + # return False + # if not isinstance(other, self.__class__): + # return False + # return self.ipSrc == other.ipSrc \ + # and self.portSrc == other.portSrc \ + # and self.ipDst == other.ipDst \ + # and self.portDst == other.portDst \ + # + # def __ne__(self, other): + # return not self.__eq__(other) + # + # def __hash__(self): + # return (str(self.ipSrc) + str(self.portSrc) + str(self.ipDst) + str(self.portDst)).__hash__() + + @abstractmethod + def getAllBytes(self): + return NotImplemented + + @abstractmethod + def getFirstBytes(self, count): + return NotImplemented + + @abstractmethod + def addPacket(self, packet, ts): + return NotImplemented \ No newline at end of file diff --git a/pcapminey/core/Streams/StreamBuilder.py b/pcapminey/core/Streams/StreamBuilder.py new file mode 100644 index 0000000000000000000000000000000000000000..9c103e0281ddddf6019a3c102ed9b1f0d744e9ca --- /dev/null +++ b/pcapminey/core/Streams/StreamBuilder.py @@ -0,0 +1,172 @@ +# -*- coding: utf8 -*- +__author__ = 'Viktor Winkelmann' + + +from TCPStream import * +from UDPStream import * +import socket +import os +import sys +import dpkt + +# Workaround to get access to pcap packet record capture length field +def myIter(self): + while True: + buf = self._Reader__f.read(dpkt.pcap.PktHdr.__hdr_len__) + if not buf: + break + hdr = self._Reader__ph(buf) + buf = self._Reader__f.read(hdr.caplen) + yield (hdr.tv_sec + (hdr.tv_usec / 1000000.0), hdr.caplen == hdr.len, buf) + +class StreamBuilder: + def __init__(self, pcapfile = None, **kwargs): + self.tcpStreams = [] + self.udpStreams = [] + self.UDP_TIMEOUT = 120 + self.VERIFY_CHECKSUMS = True # Might need to be disabled if Checksum Offloading + # was used on the capturing NIC + + if 'udpTimeout' in kwargs: + self.UDP_TIMEOUT = kwargs['udpTimeout'] + + if 'verifyChecksums' in kwargs: + self.VERIFY_CHECKSUMS = kwargs['verifyChecksums'] + + self.__parsePcapfile(pcapfile) + + + # Verify Layer3/4 Checksums, see dpkt/ip.py __str__ method + @classmethod + def __verify_checksums(cls, ippacket): + if dpkt.in_cksum(ippacket.pack_hdr() + str(ippacket.opts)) != 0: + return False + + if (ippacket.off & (dpkt.ip.IP_MF | dpkt.ip.IP_OFFMASK)) != 0: + return True + + p = str(ippacket.data) + s = dpkt.struct.pack('>4s4sxBH', ippacket.src, ippacket.dst, + ippacket.p, len(p)) + s = dpkt.in_cksum_add(0, s) + s = dpkt.in_cksum_add(s, p) + return dpkt.in_cksum_done(s) == 0 + + def __parsePcapfile(self, pcapfile): + if pcapfile is None: + return + + + with open(pcapfile, 'rb') as pcap: + dpkt.pcap.Reader.__iter__ = myIter + packets = dpkt.pcap.Reader(pcap) + caplenError = False + + fsize = float(os.path.getsize(pcapfile)) + progress = -1 + + openTcpStreams = [] + openUdpStreams = [] + + #print ' Size of file %s: %.2f mb' % (pcapfile, fsize / 1000000) + for ts, complete, rawpacket in packets: + + if not complete: + caplenError = True + + + pos = int((pcap.tell() / fsize) * 100) + if pos > progress: + # removed to use as lib need to patch + #sys.stdout.write("\r\t%d%%" % (pos,)) + #sys.stdout.flush() + progress = pos + #if progress > 15: break + + eth = dpkt.ethernet.Ethernet(rawpacket) + if eth.type != dpkt.ethernet.ETH_TYPE_IP: + continue + + ip = eth.data + + if self.VERIFY_CHECKSUMS and not self.__verify_checksums(ip): + continue + + packet = ip.data + if ip.p == dpkt.ip.IP_PROTO_TCP: + + # get last matching stream occurrence for packet + tcpStream = self.__findLastStreamOccurenceIn(openTcpStreams, + ip.src, packet.sport, + ip.dst, packet.dport) + + # no matching open stream found, create new stream if syn flag is set + if tcpStream is None: + if not packet.flags & dpkt.tcp.TH_SYN: + continue + + tcpStream = TCPStream(socket.inet_ntoa(ip.src), packet.sport, + socket.inet_ntoa(ip.dst), packet.dport) + openTcpStreams.append(tcpStream) + + # add packet to currently referenced stream + tcpStream.addPacket(packet, ts) + + # check if stream needs to be closed due to fin flag and verify stream + if packet.flags & dpkt.tcp.TH_FIN: + if tcpStream.isValid(): + tcpStream.closed = True + self.tcpStreams.append(tcpStream) + openTcpStreams.remove(tcpStream) + + + elif ip.p == dpkt.ip.IP_PROTO_UDP: + if len(packet.data) == 0: + continue + + #get last matching stream occurrence for packet + udpStream = self.__findLastStreamOccurenceIn(openUdpStreams, + ip.src, packet.sport, + ip.dst, packet.dport) + + + # no matching open stream found, create new stream + if udpStream is None or udpStream.closed: + udpStream = UDPStream(socket.inet_ntoa(ip.src), packet.sport, + socket.inet_ntoa(ip.dst), packet.dport) + openUdpStreams.append(udpStream) + + else: + lastSeen = udpStream.tsLastPacket + + # timeout happened, close old and create new stream + if lastSeen and (ts - lastSeen) > self.UDP_TIMEOUT: + udpStream.closed = True + openUdpStreams.remove(udpStream) + self.udpStreams.append(udpStream) + + udpStream = UDPStream(socket.inet_ntoa(ip.src), packet.sport, + socket.inet_ntoa(ip.dst), packet.dport) + openUdpStreams.append(udpStream) + + # add packet to currently referenced udpStream + udpStream.addPacket(packet, ts) + else: + continue + + self.tcpStreams += filter(lambda s: s.isValid(), openTcpStreams) + self.udpStreams += openUdpStreams + + #if caplenError: + #print '\nWarning: Packet loss due to too small capture length!' + + def __findLastStreamOccurenceIn(cls, list, ipSrc, portSrc, ipDst, portDst): + for stream in list[::-1]: + if stream.portSrc == portSrc \ + and stream.portDst == portDst \ + and stream.ipSrc == socket.inet_ntoa(ipSrc) \ + and stream.ipDst == socket.inet_ntoa(ipDst): + + return stream + + return None diff --git a/pcapminey/core/Streams/TCPStream.py b/pcapminey/core/Streams/TCPStream.py new file mode 100644 index 0000000000000000000000000000000000000000..05695eeadbc6deac24bf258730f6f39edfdafd3a --- /dev/null +++ b/pcapminey/core/Streams/TCPStream.py @@ -0,0 +1,75 @@ +# -*- coding: utf8 -*- +__author__ = 'Viktor Winkelmann' + +from cStringIO import StringIO +from contextlib import closing +from PacketStream import * +import dpkt + +class TCPStream(PacketStream): + def __init__(self, ipSrc, portSrc, ipDst, portDst): + PacketStream.__init__(self, ipSrc, portSrc, ipDst, portDst) + + self.packets = dict() + + def __len__(self): + return len(self.packets) + + def addPacket(self, packet, ts): + if type(packet) != dpkt.tcp.TCP: + raise TypeError('Packet is not a TCP packet!') + + if len(packet.data) == 0: + return + + if packet.seq not in self.packets: + self.packets[packet.seq] = packet + + if self.tsFirstPacket is None or ts < self.tsFirstPacket: + self.tsFirstPacket = ts + + if self.tsLastPacket is None or ts > self.tsLastPacket: + self.tsLastPacket = ts + + + def __iter__(self): + sortedPackets = sorted(self.packets.values(), key=lambda v: v.seq) + for packet in sortedPackets: + yield packet + + def getFirstBytes(self, count): + with closing(StringIO()) as bytes: + index = 0 + sortedPackets = sorted(self.packets.values(), key=lambda v: v.seq) + while len(bytes) < count and index < len(sortedPackets): + bytes.write(sortedPackets[index].data) + index += 1 + + return bytes.getvalue()[:count] + + def getAllBytes(self): + with closing(StringIO()) as bytes: + for packet in self: + bytes.write(packet.data) + + return bytes.getvalue() + + def isValid(self): + if len(self.packets) == 0: + return False + + sortedPackets = sorted(self.packets.values(), key=lambda v: v.seq) + firstPacket = sortedPackets[0] + + nextSeq = firstPacket.seq + len(firstPacket.data) + + for packet in sortedPackets[1:]: + if packet.seq != nextSeq: + return False + + if len(packet.data) == 0: + nextSeq += 1 + else: + nextSeq += len(packet.data) + + return True diff --git a/pcapminey/core/Streams/UDPStream.py b/pcapminey/core/Streams/UDPStream.py new file mode 100644 index 0000000000000000000000000000000000000000..1f96be7738e99bca646f1799a16ebf561cd804f3 --- /dev/null +++ b/pcapminey/core/Streams/UDPStream.py @@ -0,0 +1,42 @@ +# -*- coding: utf8 -*- +__author__ = 'Viktor Winkelmann' + +from PacketStream import * +import dpkt + +class UDPStream(PacketStream): + def __init__(self, ipSrc, portSrc, ipDst, portDst): + PacketStream.__init__(self, ipSrc, portSrc, ipDst, portDst) + + self.packets = [] + self.tsLastPacket = None + + def addPacket(self, packet, ts): + if type(packet) != dpkt.udp.UDP: + raise TypeError('Packet is no UDP packet!') + + if len(packet.data) == 0: + return + + if len(self.packets) == 0: + self.tsFirstpacket = ts + + self.packets.append(packet) + self.tsLastPacket = ts + + def __iter__(self): + return iter(self.packets) + + def getFirstBytes(self, count): + bytes = b'' + index = 0 + while len(bytes) < count and index < len(self.packets): + bytes += self.packets[index].data + index += 1 + return bytes[:count] + + def getAllBytes(self): + bytes = b'' + for packet in self.packets: + bytes += packet.data + return bytes diff --git a/pcapminey/core/Streams/__init__.py b/pcapminey/core/Streams/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8523449c0c5b0efecfe411746c58838e3ecb6b08 --- /dev/null +++ b/pcapminey/core/Streams/__init__.py @@ -0,0 +1,2 @@ +__author__ = 'Viktor Winkelmann' +__all__=['PacketStream', 'StreamBuilder', 'TCPStream', 'UDPStream'] \ No newline at end of file diff --git a/pcapminey/core/ThreadPool/Pool.py b/pcapminey/core/ThreadPool/Pool.py new file mode 100644 index 0000000000000000000000000000000000000000..4a0d70ed28500ae0edbad8023f3ee20304c7c976 --- /dev/null +++ b/pcapminey/core/ThreadPool/Pool.py @@ -0,0 +1,32 @@ +# -*- coding: utf8 -*- +__author__ = 'Viktor Winkelmann' + +from Queue import Queue +from Worker import Worker + +class Pool: + def __init__(self, size): + self.size = size + self.workers = [] + self.tasks = Queue() + + def _removeDeadWorkers(self): + self.workers = [w for w in self.workers if w.isAlive()] + + def map_async(self, func, objects, callback): + self._removeDeadWorkers() + if not len(self.workers) == 0: + raise Exception('ThreadPool is still working! Adding new jobs is not allowed!') + + for object in objects: + self.tasks.put((func, object, callback)) + + for id in range(self.size): + self.workers.append(Worker(id, self.tasks)) + + for worker in self.workers: + worker.start() + + def join(self): + for worker in self.workers: + worker.join() \ No newline at end of file diff --git a/pcapminey/core/ThreadPool/Worker.py b/pcapminey/core/ThreadPool/Worker.py new file mode 100644 index 0000000000000000000000000000000000000000..9493188a7fcf4c8e2f2f2db82832662d90728e10 --- /dev/null +++ b/pcapminey/core/ThreadPool/Worker.py @@ -0,0 +1,24 @@ +# -*- coding: utf8 -*- +__author__ = 'Viktor Winkelmann' + +from threading import Thread +import Queue + +class Worker(Thread): + def __init__(self, id, tasks): + Thread.__init__(self) + self.setName('Workerthread %d' % (id,)) + self.tasks = tasks + self.daemon = True + self.stop = False + + def run(self): + while not self.stop: + try: + (func, object, callback) = self.tasks.get_nowait() + callback(func(object)) + except Queue.Empty: + self.stop = True + except Exception as ex: + #print ex + self.stop = True diff --git a/pcapminey/core/ThreadPool/__init__.py b/pcapminey/core/ThreadPool/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..765a41332f98f69cf5d14e2dfe373e75ca959221 --- /dev/null +++ b/pcapminey/core/ThreadPool/__init__.py @@ -0,0 +1,2 @@ +__author__ = 'Viktor Winkelmann' +__all__ = ['Pool', 'Worker'] \ No newline at end of file diff --git a/pcapminey/core/Utils.py b/pcapminey/core/Utils.py new file mode 100644 index 0000000000000000000000000000000000000000..1579531344661a534f8bfb0f372b27844d226443 --- /dev/null +++ b/pcapminey/core/Utils.py @@ -0,0 +1,11 @@ +# -*- coding: utf8 -*- +__author__ = 'Viktor Winkelmann' + +from threading import Lock + +_printLock = Lock() + +# synchronized print +def printl(string): + with _printLock: + print string \ No newline at end of file diff --git a/pcapminey/core/Visualizations/__init__.py b/pcapminey/core/Visualizations/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..61cc9dc341e9a306e4af0d7d2f02b18b86f76eca --- /dev/null +++ b/pcapminey/core/Visualizations/__init__.py @@ -0,0 +1 @@ +__author__ = 'Viktor Winkelmann' diff --git a/pcapminey/core/__init__.py b/pcapminey/core/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..61cc9dc341e9a306e4af0d7d2f02b18b86f76eca --- /dev/null +++ b/pcapminey/core/__init__.py @@ -0,0 +1 @@ +__author__ = 'Viktor Winkelmann' diff --git a/pcapminey/dcapminey.py b/pcapminey/dcapminey.py new file mode 100755 index 0000000000000000000000000000000000000000..4f208d23809bc24d6cb5d72ae6c9076d43952d19 --- /dev/null +++ b/pcapminey/dcapminey.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python2.7 +# -*- coding: utf8 -*- + +# import pip + +# pip.main(['-q', 'install', 'cymruwhois']) +# pip.main(['-q', 'install', 'dpkt']) +# pip.main(['-q', 'install', 'simplejson']) + + + +try: + import dpkt +except: + print "Download dpkt" + +try: + import cymruwhois +except: + print "Download cymruwhois" + +try: + import simplejson as json +except: + print "Download simplejson" + + +import argparse +from core.Dispatcher import Dispatcher +from minepcaps import pcap_miner + + +VERSION = "1.0" + +parser = argparse.ArgumentParser(description='Extract files from a pcap-file.') +parser.add_argument('input', metavar='PCAP_FILE', help='the input file') +parser.add_argument('output', metavar='OUTPUT_FOLDER', help='the target folder for extraction', + nargs='?', default='output') +parser.add_argument("-e", dest='entropy', help="use entropy based rawdata extraction", + action="store_true", default=False) +parser.add_argument("-nv", dest='verifyChecksums', help="disable IP/TCP/UDP checksum verification", + action="store_false", default=True) +parser.add_argument("--T", dest='udpTimeout', help="set timeout for UDP-stream heuristics", + type=int, default=120) +args = parser.parse_args() +readyPath = args.input +miner = pcap_miner(readyPath) +jsonResults = miner.summary2json() +pyResults = json.loads(jsonResults) +#print pyResults + + +#print 'pcapfex - Packet Capture Forensic Evidence Extractor - version %s' % (VERSION,) +#print '----------=------===-----=--------=---------=------------------' + '-'*len(VERSION) + '\n' + +if not args.verifyChecksums: + pyResults['verifiyChecksums'] = 'Packet checksum verification disabled.' +if args.entropy: + pyResults['entropySetting'] = 'Using entropy and statistical analysis for raw extraction and classification of unknown data.' + + +dispatcher = Dispatcher(args.input, args.output, args.entropy, + verifyChecksums=args.verifyChecksums, + udpTimeout=args.udpTimeout, + ) +results = dispatcher.run() +pyResults['files_found'] = results.filenamelist +print json.dumps(pyResults) + +if(pyResults["counts"]): + displayData = tableToMarkdown('PCAP Data Frequency Counts', pyResults["counts"]) +if(pyResults["destination_ip_details"]): + displayData += tableToMarkdown('Destination IP Details', pyResults["destination_ip_details"]) +if(pyResults["dns_data"]): + displayData += tableToMarkdown('DNS Details', pyResults["dns_data"]) +if(pyResults["http_requests"]): + displayData += tableToMarkdown('Http Requests', pyResults["http_requests"]) +if(pyResults["flows"]): + displayData += tableToMarkdown('Flow Data', pyResults["flows"]) + +demisto.results({'Type': entryTypes['note'], 'Contents': pyResults, 'EntryContext': {'pcap_results': pyResults}, 'ContentsFormat': formats['json'], 'HumanReadable': displayData}) + + + + + + diff --git a/pcapminey/minepcaps.py b/pcapminey/minepcaps.py new file mode 100644 index 0000000000000000000000000000000000000000..63da8b6487c4c58823569ee16f1c4fa312a5fc4f --- /dev/null +++ b/pcapminey/minepcaps.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python2.7 +# -*- coding: utf8 -*- + +import socket +import itertools +import operator +# import pip + + +import argparse +from core.Dispatcher import Dispatcher + +VERSION = "1.0" + +# pip.main(['-q', 'install', 'cymruwhois']) +# pip.main(['-q', 'install', 'dpkt']) +# pip.main(['-q', 'install', 'simplejson']) + + + +try: + import dpkt +except: + print "Download dpkt" + +try: + import cymruwhois +except: + print "Download cymruwhois" + +try: + import simplejson as json +except: + print "Download simplejson" + +import time, re, optparse, getpass, time + +class pcap_miner(): + def __init__(self,pcap_file): + #declaration + self._pcap_file = pcap_file + self._http_request_data = [] + self._source_ips = [] + self._destination_ips = [] + self._source_ip_details = [] + self._destination_ip_details = [] + self._dns_request_data = [] + self._flows = [] + self._packet_count = 0 + self._http_count = 0 + self._dns_count = 0 + + #processing + self._handle = self._get_dpkt_handle() + self._extract_data() + + def _get_dpkt_handle(self): + f = open(self._pcap_file) + pcap = dpkt.pcap.Reader(f) + return pcap + + def unpack_ip(self,packed_ip): + ip = socket.inet_ntoa(packed_ip) + return ip + + def quick_unique(self,seq): + seen = set() + return [ x for x in seq if x not in seen and not seen.add(x)] + + def _extract_data(self): + pcap = self._handle + eth = None + ip = None + protocol = None + + for ts, buf in pcap: + try: + eth = dpkt.ethernet.Ethernet(buf) + ip = eth.data + protocol = ip.data + except dpkt.dpkt.NeedData: + continue + + self._packet_count += 1 + + try: + source_ip = self.unpack_ip(ip.src) + destination_ip = self.unpack_ip(ip.dst) + self._source_ips.append(source_ip) + self._destination_ips.append(destination_ip) + except Exception, e: + continue + + try: + if protocol.dport == 80 or protocol.dport == 443: + self._http_count += 1 + try: + http = dpkt.http.Request(protocol.data) + tmp = http.headers + tmp["source_ip"] = source_ip + tmp['destination_ip'] = destination_ip + tmp['method'] = http.method + tmp['version'] = http.version + tmp['uri'] = http.uri + self._http_request_data.append(tmp) + except Exception, e: + continue + except Exception, e: + continue + + try: + if protocol.dport == 53 or protocol.sport == 53: + self._dns_count += 1 + try: + dns = dns = dpkt.dns.DNS(protocol.data) + if dns.qr != dpkt.dns.DNS_R: continue + if dns.opcode != dpkt.dns.DNS_QUERY: continue + if dns.rcode != dpkt.dns.DNS_RCODE_NOERR: continue + if len(dns.an) < 1: continue + for answer in dns.an: + if answer.type == 5: + tmp = { "type": "CNAME", "request":answer.name, "response":answer.cname } + self._dns_request_data.append(tmp) + elif answer.type == 1: + tmp = { "type": "A", "request":answer.name, "response":socket.inet_ntoa(answer.rdata) } + self._dns_request_data.append(tmp) + elif answer.type == 12: + tmp = { "type": "PTR", "request":answer.name, "response":answer.ptrname } + self._dns_request_data.append(tmp) + + except Exception, e: + continue + except Exception, e: + continue + try: + self._flows.append(source_ip + "/" + str(protocol.sport) + "=>" + destination_ip + "/" + str(protocol.dport)) + except Exception, e: + continue + + def get_source_ips(self): + return self.quick_unique(self._source_ips) + + def get_source_ip_details(self): + ulist = self.quick_unique(self._source_ips) + c = cymruwhois.Client() + for item in c.lookupmany(ulist): + try: + if item.prefix == None: + tmp = { "ip_address": item.ip, "block": "", "asn": "", "owner": "" } + else: + tmp = { "ip_address": item.ip, "block": item.prefix, "asn": item.asn, "owner": item.owner } + self._source_ip_details.append(tmp) + except Exception, e: + continue + + return self._source_ip_details + + def get_destination_ips(self): + return self.quick_unique(self._destination_ips) + + def get_destination_ip_details(self): + ulist = self.quick_unique(self._destination_ips) + c = cymruwhois.Client() + for item in c.lookupmany(ulist): + try: + if item.prefix == None: + tmp = { "ip_address": item.ip, "block": "", "asn": "", "owner": "" } + else: + tmp = { "ip_address": item.ip, "block": item.prefix, "asn": item.asn, "owner": item.owner } + self._destination_ip_details.append(tmp) + except Exception, e: + continue + + return self._destination_ip_details + + def get_http_request_data(self): + getvals = operator.itemgetter('source_ip','destination_ip', 'uri') + self._http_request_data.sort(key=getvals) + + result = [] + for k, g in itertools.groupby(self._http_request_data, getvals): + result.append(g.next()) + + self._http_request_data[:] = result + return self._http_request_data + + def get_dns_request_data(self): + getvals = operator.itemgetter('type','request', 'response') + self._dns_request_data.sort(key=getvals) + + result = [] + for k, g in itertools.groupby(self._dns_request_data, getvals): + result.append(g.next()) + + self._dns_request_data[:] = result + return self._dns_request_data + + def get_flows(self): + return self.quick_unique(self._flows) + + def get_packet_count(self): + return self._packet_count + + def get_http_count(self): + return self._http_count + + def get_dns_count(self): + return self._dns_count + + def summary2json(self): + self._dns_json = [] + self._dest_json = [] + self._http_json = [] + self._flow_json = [] + self._counts_json = [] + + self._counts_json.append({"packet_count":self._packet_count,"http_count":self._http_count,"dns_count":self._dns_count}) + + for dns in self.get_dns_request_data(): + self._dns_json.append({"type": dns["type"], "request": dns["request"], "response": dns["response"]}) + + for ip in self.get_destination_ip_details(): + self._dest_json.append({"ip_address" : ip['ip_address'], "owner": ip['owner'], "asn": ip['asn'], "netblock": ip['block']}) + + for info in self.get_http_request_data(): + tmp = {} + for key, value in info.items(): + tmp[key] = value + self._http_json.append(tmp) + + for flow in self.get_flows(): + direction = flow.split("=>") + source = direction[0].split("/") + destination = direction[1].split("/") + self._flow_json.append({"source_ip":source[0],"source_port":source[1],"destination_ip":destination[0],"destination_port":destination[1]}) + + #construct the output object + obj = {"file":self._pcap_file, + "dns_data":self._dns_json, + "destination_ip_details":self._dest_json, + "http_requests":self._http_json, + "flows":self._flow_json, + "counts":self._counts_json + } + + return json.dumps(obj) diff --git a/pcapminey/pcapfex.py b/pcapminey/pcapfex.py new file mode 100755 index 0000000000000000000000000000000000000000..e2398dcb644abe64cc8615b8818b0858c7081cb2 --- /dev/null +++ b/pcapminey/pcapfex.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python2.7 +# -*- coding: utf8 -*- +__author__ = 'Viktor Winkelmann' + +import argparse +from core.Dispatcher import Dispatcher + +VERSION = "1.0" + +parser = argparse.ArgumentParser(description='Extract files from a pcap-file.') +parser.add_argument('input', metavar='PCAP_FILE', help='the input file') +parser.add_argument('output', metavar='OUTPUT_FOLDER', help='the target folder for extraction', + nargs='?', default='output') +parser.add_argument("-e", dest='entropy', help="use entropy based rawdata extraction", + action="store_true", default=False) +parser.add_argument("-nv", dest='verifyChecksums', help="disable IP/TCP/UDP checksum verification", + action="store_false", default=True) +parser.add_argument("--T", dest='udpTimeout', help="set timeout for UDP-stream heuristics", + type=int, default=120) + + +print 'pcapfex - Packet Capture Forensic Evidence Extractor - version %s' % (VERSION,) +print '----------=------===-----=--------=---------=------------------' + '-'*len(VERSION) + '\n' +args = parser.parse_args() + +if not args.verifyChecksums: + print 'Packet checksum verification disabled.' +if args.entropy: + print 'Using entropy and statistical analysis for raw extraction and classification of unknown data.' + + +dispatcher = Dispatcher(args.input, args.output, args.entropy, + verifyChecksums=args.verifyChecksums, + udpTimeout=args.udpTimeout, + ) +results = dispatcher.run() +print results.filenamelist diff --git a/pcapminey/pcapminey.py b/pcapminey/pcapminey.py new file mode 100755 index 0000000000000000000000000000000000000000..79f6b59a97e60fcaa9af00b41a9f5200a6f955d9 --- /dev/null +++ b/pcapminey/pcapminey.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python2.7 +# -*- coding: utf8 -*- + +#import pip + +#pip.main(['-q', 'install', 'cymruwhois']) +#pip.main(['-q', 'install', 'dpkt']) +#pip.main(['-q', 'install', 'simplejson']) + + + +try: + import dpkt +except: + print "Download dpkt" + +try: + import cymruwhois +except: + print "Download cymruwhois" + +try: + import simplejson as json +except: + print "Download simplejson" + + +import argparse +from core.Dispatcher import Dispatcher +from minepcaps import pcap_miner + + +VERSION = "1.0" + +parser = argparse.ArgumentParser(description='Extract files from a pcap-file.') +parser.add_argument('input', metavar='PCAP_FILE', help='the input file') +parser.add_argument('output', metavar='OUTPUT_FOLDER', help='the target folder for extraction', + nargs='?', default='output') +parser.add_argument("-e", dest='entropy', help="use entropy based rawdata extraction", + action="store_true", default=False) +parser.add_argument("-nv", dest='verifyChecksums', help="disable IP/TCP/UDP checksum verification", + action="store_false", default=True) +parser.add_argument("--T", dest='udpTimeout', help="set timeout for UDP-stream heuristics", + type=int, default=120) +args = parser.parse_args() +readyPath = args.input +miner = pcap_miner(readyPath) +jsonResults = miner.summary2json() +pyResults = json.loads(jsonResults) +#print pyResults + + +#print 'pcapfex - Packet Capture Forensic Evidence Extractor - version %s' % (VERSION,) +#print '----------=------===-----=--------=---------=------------------' + '-'*len(VERSION) + '\n' + +if not args.verifyChecksums: + pyResults['verifiyChecksums'] = 'Packet checksum verification disabled.' +if args.entropy: + pyResults['entropySetting'] = 'Using entropy and statistical analysis for raw extraction and classification of unknown data.' + + +dispatcher = Dispatcher(args.input, args.output, args.entropy, + verifyChecksums=args.verifyChecksums, + udpTimeout=args.udpTimeout, + ) +results = dispatcher.run() +pyResults['files_found'] = results.filenamelist +print json.dumps(pyResults) diff --git a/pcapminey/plugins/__init__.py b/pcapminey/plugins/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..61cc9dc341e9a306e4af0d7d2f02b18b86f76eca --- /dev/null +++ b/pcapminey/plugins/__init__.py @@ -0,0 +1 @@ +__author__ = 'Viktor Winkelmann' diff --git a/pcapminey/plugins/data_recognizers/__init__.py b/pcapminey/plugins/data_recognizers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..61cc9dc341e9a306e4af0d7d2f02b18b86f76eca --- /dev/null +++ b/pcapminey/plugins/data_recognizers/__init__.py @@ -0,0 +1 @@ +__author__ = 'Viktor Winkelmann' diff --git a/pcapminey/plugins/data_recognizers/avi.py b/pcapminey/plugins/data_recognizers/avi.py new file mode 100644 index 0000000000000000000000000000000000000000..fe5092d2ec2753246eadbfce6e62b422c170c3ef --- /dev/null +++ b/pcapminey/plugins/data_recognizers/avi.py @@ -0,0 +1,18 @@ +# -*- coding: utf8 -*- +__author__ = 'Viktor Winkelmann' + +import sys + +sys.path.append('../..') +from core.Plugins.DataRecognizer import * + + +def getClassReference(): + return AviFile + + +class AviFile(DataRecognizer): + signatures = [('RIFF.{4}AVI LIST', None)] + fileEnding = "avi" + dataType = "AVI file" + dataCategory = DataCategory.VIDEO \ No newline at end of file diff --git a/pcapminey/plugins/data_recognizers/bmp.py b/pcapminey/plugins/data_recognizers/bmp.py new file mode 100644 index 0000000000000000000000000000000000000000..8eb36a565a9823abcdb4c6028894299bbfe1892f --- /dev/null +++ b/pcapminey/plugins/data_recognizers/bmp.py @@ -0,0 +1,18 @@ +# -*- coding: utf8 -*- +__author__ = 'Viktor Winkelmann' + +import sys + +sys.path.append('../..') +from core.Plugins.DataRecognizer import * + + +def getClassReference(): + return BMPFile + + +class BMPFile(DataRecognizer): + signatures = [(b'\x42\x4D.{4}\x00\x00\x00\x00', None)] + fileEnding = "bmp" + dataType = "Bitmap file" + dataCategory = DataCategory.IMAGE diff --git a/pcapminey/plugins/data_recognizers/elf.py b/pcapminey/plugins/data_recognizers/elf.py new file mode 100644 index 0000000000000000000000000000000000000000..2d0a15192f3cfedbd4c336c4e069f324ca5b7266 --- /dev/null +++ b/pcapminey/plugins/data_recognizers/elf.py @@ -0,0 +1,18 @@ +# -*- coding: utf8 -*- +__author__ = 'Viktor Winkelmann' + +import sys + +sys.path.append('../..') +from core.Plugins.DataRecognizer import * + + +def getClassReference(): + return ELFFile + + +class ELFFile(DataRecognizer): + signatures = [(b'\x7F\x45\x4C\x46', None)] + fileEnding = "" + dataType = "ELF file" + dataCategory = DataCategory.EXECUTABLE diff --git a/pcapminey/plugins/data_recognizers/exe.py b/pcapminey/plugins/data_recognizers/exe.py new file mode 100644 index 0000000000000000000000000000000000000000..4648f217105e86bab9b43a7f0713a9497af1b230 --- /dev/null +++ b/pcapminey/plugins/data_recognizers/exe.py @@ -0,0 +1,18 @@ +# -*- coding: utf8 -*- +__author__ = 'Viktor Winkelmann' + +import sys + +sys.path.append('../..') +from core.Plugins.DataRecognizer import * + + +def getClassReference(): + return EXEFile + + +class EXEFile(DataRecognizer): + signatures = [('MZ.{254}PE', None)] + fileEnding = "exe" + dataType = "Windows PE file" + dataCategory = DataCategory.EXECUTABLE diff --git a/pcapminey/plugins/data_recognizers/gif.py b/pcapminey/plugins/data_recognizers/gif.py new file mode 100644 index 0000000000000000000000000000000000000000..83ac0bf08f23aa497eddc1b01771b32c24d5f846 --- /dev/null +++ b/pcapminey/plugins/data_recognizers/gif.py @@ -0,0 +1,18 @@ +# -*- coding: utf8 -*- +__author__ = 'Viktor Winkelmann' + +import sys + +sys.path.append('../..') +from core.Plugins.DataRecognizer import * + + +def getClassReference(): + return GIFFile + + +class GIFFile(DataRecognizer): + signatures = [(b'\x47\x49\x46\x38[\x37\x39]\x61', b'\x00\x3B')] + fileEnding = "gif" + dataType = "GIF file" + dataCategory = DataCategory.IMAGE diff --git a/pcapminey/plugins/data_recognizers/jpeg.py b/pcapminey/plugins/data_recognizers/jpeg.py new file mode 100644 index 0000000000000000000000000000000000000000..f4037a4d1686d12d266297304401b14ef5660f34 --- /dev/null +++ b/pcapminey/plugins/data_recognizers/jpeg.py @@ -0,0 +1,18 @@ +# -*- coding: utf8 -*- +__author__ = 'Viktor Winkelmann' + +import sys + +sys.path.append('../..') +from core.Plugins.DataRecognizer import * + + +def getClassReference(): + return JpegFile + + +class JpegFile(DataRecognizer): + signatures = [(b'\xFF\xD8\xFF', b'\xFF\xD9'), (b'.{6}\x4A\x46\x49\x46\x00', None)] + fileEnding = "jpg" + dataType = "JPEG file" + dataCategory = DataCategory.IMAGE diff --git a/pcapminey/plugins/data_recognizers/mkv.py b/pcapminey/plugins/data_recognizers/mkv.py new file mode 100644 index 0000000000000000000000000000000000000000..64ad1a0e9a4379ede9b52f9c06c8baf7301750b1 --- /dev/null +++ b/pcapminey/plugins/data_recognizers/mkv.py @@ -0,0 +1,18 @@ +# -*- coding: utf8 -*- +__author__ = 'Viktor Winkelmann' + +import sys + +sys.path.append('../..') +from core.Plugins.DataRecognizer import * + + +def getClassReference(): + return MKVFile + + +class MKVFile(DataRecognizer): + signatures = [(b'\x1A\x45\xDF\xA3.{4}matroska', None)] + fileEnding = "mpg" + dataType = "MKV file" + dataCategory = DataCategory.VIDEO \ No newline at end of file diff --git a/pcapminey/plugins/data_recognizers/mp3.py b/pcapminey/plugins/data_recognizers/mp3.py new file mode 100644 index 0000000000000000000000000000000000000000..5b681152400942adf03dbd7cd13bbfdfb7fbd52a --- /dev/null +++ b/pcapminey/plugins/data_recognizers/mp3.py @@ -0,0 +1,18 @@ +# -*- coding: utf8 -*- +__author__ = 'Viktor Winkelmann' + +import sys + +sys.path.append('../..') +from core.Plugins.DataRecognizer import * + + +def getClassReference(): + return MP3File + + +class MP3File(DataRecognizer): + signatures = [('ID3', None)] + fileEnding = "mp3" + dataType = "MP3 file" + dataCategory = DataCategory.AUDIO \ No newline at end of file diff --git a/pcapminey/plugins/data_recognizers/mpg.py b/pcapminey/plugins/data_recognizers/mpg.py new file mode 100644 index 0000000000000000000000000000000000000000..02ad1fc5f1ea03380ac72978334aef65d25225c6 --- /dev/null +++ b/pcapminey/plugins/data_recognizers/mpg.py @@ -0,0 +1,18 @@ +# -*- coding: utf8 -*- +__author__ = 'Viktor Winkelmann' + +import sys + +sys.path.append('../..') +from core.Plugins.DataRecognizer import * + + +def getClassReference(): + return MPGFile + + +class MPGFile(DataRecognizer): + signatures = [(b'\x00\x00\x01[\xB0-\xBF]', b'\x00\x00\x01[\xB7\xB9]')] + fileEnding = "mpg" + dataType = "MPEG file" + dataCategory = DataCategory.VIDEO \ No newline at end of file diff --git a/pcapminey/plugins/data_recognizers/pdf.py b/pcapminey/plugins/data_recognizers/pdf.py new file mode 100644 index 0000000000000000000000000000000000000000..df4168c6a5674c5d32adfb80448728f6452fd7a5 --- /dev/null +++ b/pcapminey/plugins/data_recognizers/pdf.py @@ -0,0 +1,18 @@ +# -*- coding: utf8 -*- +__author__ = 'Viktor Winkelmann' + +import sys + +sys.path.append('../..') +from core.Plugins.DataRecognizer import * + + +def getClassReference(): + return PDFFile + + +class PDFFile(DataRecognizer): + signatures = [(b'\x25\x50\x44\x46', b'.*\x25\x25\x45\x4F\x46[(\x0A)(\x0D)(\x0D\x0A)]?')] + fileEnding = "pdf" + dataType = "PDF file" + dataCategory = DataCategory.DOC diff --git a/pcapminey/plugins/data_recognizers/png.py b/pcapminey/plugins/data_recognizers/png.py new file mode 100644 index 0000000000000000000000000000000000000000..b0dc0410029a567e8417261e3245c32330735f42 --- /dev/null +++ b/pcapminey/plugins/data_recognizers/png.py @@ -0,0 +1,18 @@ +# -*- coding: utf8 -*- +__author__ = 'Viktor Winkelmann' + +import sys + +sys.path.append('../..') +from core.Plugins.DataRecognizer import * + + +def getClassReference(): + return PNGFile + + +class PNGFile(DataRecognizer): + signatures = [(b'\x89\x50\x4E\x47\x0D\x0A\x1A\x0A', b'\x49\x45\x4E\x44\xAE\x42\x60\x82')] + fileEnding = "png" + dataType = "PNG file" + dataCategory = DataCategory.IMAGE diff --git a/pcapminey/plugins/data_recognizers/rar.py b/pcapminey/plugins/data_recognizers/rar.py new file mode 100644 index 0000000000000000000000000000000000000000..20ad88331ba928470955f338af2dfc98ed46e63b --- /dev/null +++ b/pcapminey/plugins/data_recognizers/rar.py @@ -0,0 +1,18 @@ +# -*- coding: utf8 -*- +__author__ = 'Viktor Winkelmann' + +import sys + +sys.path.append('../..') +from core.Plugins.DataRecognizer import * + + +def getClassReference(): + return RARFile + + +class RARFile(DataRecognizer): + signatures = [(b'\x52\x61\x72\x21\x1A\x07', None)] + fileEnding = "rar" + dataType = "RAR file" + dataCategory = DataCategory.COMPRESSED \ No newline at end of file diff --git a/pcapminey/plugins/data_recognizers/wav.py b/pcapminey/plugins/data_recognizers/wav.py new file mode 100644 index 0000000000000000000000000000000000000000..db13d4cb77bd75cbe1d39e057447a39a3d96e830 --- /dev/null +++ b/pcapminey/plugins/data_recognizers/wav.py @@ -0,0 +1,18 @@ +# -*- coding: utf8 -*- +__author__ = 'Viktor Winkelmann' + +import sys + +sys.path.append('../..') +from core.Plugins.DataRecognizer import * + + +def getClassReference(): + return WAVFile + + +class WAVFile(DataRecognizer): + signatures = [(b'RIFF.{4}WAVEfmt', None)] + fileEnding = "wav" + dataType = "WAV file" + dataCategory = DataCategory.AUDIO \ No newline at end of file diff --git a/pcapminey/plugins/data_recognizers/zip.py b/pcapminey/plugins/data_recognizers/zip.py new file mode 100644 index 0000000000000000000000000000000000000000..da443c9f28b278dbf5c44fc01e240be9aec06ad8 --- /dev/null +++ b/pcapminey/plugins/data_recognizers/zip.py @@ -0,0 +1,18 @@ +# -*- coding: utf8 -*- +__author__ = 'Viktor Winkelmann' + +import sys + +sys.path.append('../..') +from core.Plugins.DataRecognizer import * + + +def getClassReference(): + return ZipFile + + +class ZipFile(DataRecognizer): + signatures = [(b'\x50\x4B\x03\x04', None)] + fileEnding = "zip" + dataType = "ZIP file" + dataCategory = DataCategory.COMPRESSED \ No newline at end of file diff --git a/pcapminey/plugins/decoders/__init__.py b/pcapminey/plugins/decoders/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..61cc9dc341e9a306e4af0d7d2f02b18b86f76eca --- /dev/null +++ b/pcapminey/plugins/decoders/__init__.py @@ -0,0 +1 @@ +__author__ = 'Viktor Winkelmann' diff --git a/pcapminey/plugins/decoders/base64data.py b/pcapminey/plugins/decoders/base64data.py new file mode 100644 index 0000000000000000000000000000000000000000..515ca8ff66d679ca85392146c0b16575952fd903 --- /dev/null +++ b/pcapminey/plugins/decoders/base64data.py @@ -0,0 +1,28 @@ +# -*- coding: utf8 -*- +__author__ = 'Viktor Winkelmann' + +import sys +import base64 + +sys.path.append('../..') +from core.Plugins.Decoder import * + + +def getClassReference(): + return Base64Decoder + + +class Base64Decoder(Decoder): + decoderName = "base64" + + @classmethod + def decodeData(cls, data): + #allowedChars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/\r\n\t " + #for char in data: + # if char not in allowedChars: + # return None + + try: + return base64.b64decode(data) + except TypeError: + return None diff --git a/pcapminey/plugins/decoders/plaindata.py b/pcapminey/plugins/decoders/plaindata.py new file mode 100644 index 0000000000000000000000000000000000000000..479a8f9734451ce5414e9d27f49efb6c16b10d78 --- /dev/null +++ b/pcapminey/plugins/decoders/plaindata.py @@ -0,0 +1,21 @@ +# -*- coding: utf8 -*- +__author__ = 'Viktor Winkelmann' + +import sys + +sys.path.append('../..') +from core.Plugins.Decoder import * + + +def getClassReference(): + return PlainData + + +class PlainData(Decoder): + basePriority = 10 + + decoderName = "plain data" + + @classmethod + def decodeData(cls, data): + return data \ No newline at end of file diff --git a/pcapminey/plugins/protocol_dissectors/__init__.py b/pcapminey/plugins/protocol_dissectors/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..61cc9dc341e9a306e4af0d7d2f02b18b86f76eca --- /dev/null +++ b/pcapminey/plugins/protocol_dissectors/__init__.py @@ -0,0 +1 @@ +__author__ = 'Viktor Winkelmann' diff --git a/pcapminey/plugins/protocol_dissectors/http10.py b/pcapminey/plugins/protocol_dissectors/http10.py new file mode 100644 index 0000000000000000000000000000000000000000..5f5f407daa6dcac4d9e8f1973cfd56a452199310 --- /dev/null +++ b/pcapminey/plugins/protocol_dissectors/http10.py @@ -0,0 +1,41 @@ +# -*- coding: utf8 -*- +__author__ = 'Viktor Winkelmann' + +import sys +sys.path.append('../..') + +from plugins.protocol_dissectors.http11 import HTTP11 +from cStringIO import StringIO +from contextlib import closing + +def getClassReference(): + return HTTP10 + +# Parses HTTP Requests / Responses according to http://tools.ietf.org/html/rfc1945 +class HTTP10(HTTP11): + basePriority = 45 + + protocolName = "HTTP 1.0" + + @classmethod + def parseData(cls, data): + with closing(StringIO(data)) as data: + # check start line for HTTP 1.0 tag + line = data.readline() + if 'HTTP/1.0' not in line: + return None + + # classify as Request or Response + if line.startswith('HTTP'): + return cls.getResponsePayload(data) + else: + return cls.getRequestPayload(data) + + + + + + + + + diff --git a/pcapminey/plugins/protocol_dissectors/http11.py b/pcapminey/plugins/protocol_dissectors/http11.py new file mode 100644 index 0000000000000000000000000000000000000000..fa5f75fd4a8fadeb4fc222c223a0ae6cd229992c --- /dev/null +++ b/pcapminey/plugins/protocol_dissectors/http11.py @@ -0,0 +1,105 @@ +# -*- coding: utf8 -*- +__author__ = 'Viktor Winkelmann' + +import sys +sys.path.append('../..') + +from core.Plugins.ProtocolDissector import * +from cStringIO import StringIO +from contextlib import closing +from gzip import GzipFile + +def getClassReference(): + return HTTP11 + + +# Parses HTTP Requests / Responses according to http://tools.ietf.org/html/rfc7230 +class HTTP11(ProtocolDissector): + defaultPorts = [80, 8080, 8000, 443] + + decoders = { + 'gzip': lambda x: GzipFile(fileobj=StringIO(x)).read(), + 'x-gzip': lambda x: GzipFile(fileobj=StringIO(x)).read(), + 'deflate': lambda x: x.decode('zlib'), + } + + protocolName = "HTTP 1.1" + + @classmethod + def getRequestPayload(cls, data): + return cls.getResponsePayload(data) # No special case found yet that has to be handled differently + + @classmethod + def decode(cls, payload, encoding): + if not payload: + return None + + if encoding not in cls.decoders.keys(): + return payload + + try: + return cls.decoders[encoding](payload) + except Exception as e: + return payload + + @classmethod + def getResponsePayload(cls, data): + payload = None + encoding = None + headers = cls.parseHeaders(data) + if 'Content-Length' in headers: + length = int(headers['Content-Length']) + payload = data.read(length) + + if 'Content-Encoding' in headers: + encoding = headers['Content-Encoding'] + encoding = encoding.split(':')[-1].strip().lower() + + if 'Transfer-Encoding' in headers: + encoding = headers['Transfer-Encoding'] + encoding = encoding.split(':')[-1].strip().lower() + + return cls.decode(payload, encoding) + + @classmethod + def parseHeaders(cls, data): + headers = dict() + line = data.readline() + while line not in ['\r\n','']: + keyval = line.split(':') + headers[keyval[0].strip()] = keyval[1].strip() + line = data.readline() + return headers + + @classmethod + def parseData(cls, data): + with closing(StringIO(data)) as data: + # check start line for HTTP 1.1 tag + line = data.readline() + if 'HTTP/1.1' not in line: + return None + + payloads = [] + + #loop to allow HTTP pipelining + while line != '': + # classify as Request or Response + if line.startswith('HTTP'): + payload = cls.getResponsePayload(data) + else: + payload = cls.getRequestPayload(data) + + if payload: + payloads.append(payload) + + line = data.readline() + return payloads + + + + + + + + + diff --git a/pcapminey/plugins/protocol_dissectors/unknown.py b/pcapminey/plugins/protocol_dissectors/unknown.py new file mode 100644 index 0000000000000000000000000000000000000000..4df54bfc0539d63d530cc2aba53dac32af9ff90a --- /dev/null +++ b/pcapminey/plugins/protocol_dissectors/unknown.py @@ -0,0 +1,21 @@ +# -*- coding: utf8 -*- +__author__ = 'Viktor Winkelmann' + +import sys + +sys.path.append('../..') +from core.Plugins.ProtocolDissector import * + + +def getClassReference(): + return Unknown + + +class Unknown(ProtocolDissector): + basePriority = 200 + + protocolName = "unknown protocol" + + @classmethod + def parseData(cls, data): + return [data] diff --git a/pcapminey/runAgainstAllPCAP.sh b/pcapminey/runAgainstAllPCAP.sh new file mode 100755 index 0000000000000000000000000000000000000000..60af9cd871ca784614b5e0f732a054a7c0d2928a --- /dev/null +++ b/pcapminey/runAgainstAllPCAP.sh @@ -0,0 +1,8 @@ +#!/bin/bash +FILES=./THREAT_PCAP/* +for f in $FILES +do + echo "Processing $f file..." + # take action on each file. $f store current file name + python pcapfex.py $f +done diff --git a/pcapminey/tests/entropy/testEntropyClassifier.py b/pcapminey/tests/entropy/testEntropyClassifier.py new file mode 100644 index 0000000000000000000000000000000000000000..51fbd2518c4eaa06f98ee55b805ccb98753662ec --- /dev/null +++ b/pcapminey/tests/entropy/testEntropyClassifier.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python2.7 +# -*- coding: utf8 -*- +__author__ = 'Viktor Winkelmann' + +import sys + +sys.path.append('../..') +import unittest, zlib, random +from core.Plugins.EntropyClassifier import * +from Crypto.Cipher import AES, ARC4 + +class TestEntropyClassifier(unittest.TestCase): + def setUp(self): + self.text = ''' + Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore + et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. + Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, + consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, + sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea + takimata sanctus est Lorem ipsum dolor sit amet. + ''' * 16 + + randomData = ''.join(chr(random.randint(0, 255)) for _ in range(5000)) + + self.zip6 = zlib.compress(self.text + randomData, 6) + self.zip9 = zlib.compress(self.text + randomData, 9) + + pwd = '0123456789ABCDEF' + aes = AES.new(pwd, AES.MODE_CBC, IV='\x42'*16) + arc4 = ARC4.new(pwd) + print len(self.text) + self.enc1 = aes.encrypt(self.text) + self.enc2 = arc4.encrypt(self.text) + + + def test_classifier(self): + self.assertRaises(DataLengthException, EntropyClassifier.classify, 'Too short') + self.assertEqual(EntropyClass.PLAIN, EntropyClassifier.classify(self.text)) + self.assertEqual(EntropyClass.COMPRESSED, EntropyClassifier.classify(self.zip6)) + self.assertEqual(EntropyClass.COMPRESSED, EntropyClassifier.classify(self.zip9)) + self.assertEqual(EntropyClass.ENCRYPTED, EntropyClassifier.classify(self.enc1)) + self.assertEqual(EntropyClass.ENCRYPTED, EntropyClassifier.classify(self.enc2)) + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/pcapminey/tests/scenarios/4.1/4.1.pcap b/pcapminey/tests/scenarios/4.1/4.1.pcap new file mode 100644 index 0000000000000000000000000000000000000000..6935a1e12a65ec341454d898f4acb9af18442c48 Binary files /dev/null and b/pcapminey/tests/scenarios/4.1/4.1.pcap differ diff --git a/pcapminey/tests/scenarios/4.1/file.zip b/pcapminey/tests/scenarios/4.1/file.zip new file mode 100644 index 0000000000000000000000000000000000000000..63db28a25fe083416d594214d23dfd15dba5b00f Binary files /dev/null and b/pcapminey/tests/scenarios/4.1/file.zip differ diff --git a/pcapminey/tests/scenarios/4.2/4.2.pcap b/pcapminey/tests/scenarios/4.2/4.2.pcap new file mode 100644 index 0000000000000000000000000000000000000000..0cff38d4b4bd654b853023021a209ba68ed23a13 Binary files /dev/null and b/pcapminey/tests/scenarios/4.2/4.2.pcap differ diff --git a/pcapminey/tests/scenarios/4.2/file.jpg b/pcapminey/tests/scenarios/4.2/file.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f98e686ec89ee0ef49c26b5766e9dfac16d81b4d Binary files /dev/null and b/pcapminey/tests/scenarios/4.2/file.jpg differ diff --git a/pcapminey/tests/scenarios/4.3/4.3.pcap b/pcapminey/tests/scenarios/4.3/4.3.pcap new file mode 100644 index 0000000000000000000000000000000000000000..76938aea7eaa06c27dfa225573a60b8ce53c3905 Binary files /dev/null and b/pcapminey/tests/scenarios/4.3/4.3.pcap differ diff --git a/pcapminey/tests/scenarios/4.3/file.pdf b/pcapminey/tests/scenarios/4.3/file.pdf new file mode 100644 index 0000000000000000000000000000000000000000..4c546535f32051d3cd9e7cb75bae588c04a57cb6 Binary files /dev/null and b/pcapminey/tests/scenarios/4.3/file.pdf differ diff --git a/pcapminey/tests/scenarios/4.4/4.4.pcap b/pcapminey/tests/scenarios/4.4/4.4.pcap new file mode 100644 index 0000000000000000000000000000000000000000..a92fdfe0a5cd54ac569a50722144381dcc0320cd Binary files /dev/null and b/pcapminey/tests/scenarios/4.4/4.4.pcap differ diff --git a/pcapminey/tests/scenarios/4.4/file.mp3 b/pcapminey/tests/scenarios/4.4/file.mp3 new file mode 100644 index 0000000000000000000000000000000000000000..084a7d1f8d13f74530095ab463c4b0b3bd809d97 Binary files /dev/null and b/pcapminey/tests/scenarios/4.4/file.mp3 differ diff --git a/pcapminey/tests/scenarios/4.4/sendfile.py b/pcapminey/tests/scenarios/4.4/sendfile.py new file mode 100644 index 0000000000000000000000000000000000000000..be37036fbaeaf75f6fdc708a64b4524f050fdb75 --- /dev/null +++ b/pcapminey/tests/scenarios/4.4/sendfile.py @@ -0,0 +1,14 @@ +# -*- coding: utf8 -*- +__author__ = 'Viktor Winkelmann' + +import base64 +from socket import socket + +with open('file.mp3', 'rb') as inputfile: + data = inputfile.read() + b64data = base64.b64encode(data) + + s = socket() + s.connect(('192.168.123.37', 4242)) + s.sendall(b64data) + s.close() diff --git a/pcapminey/tests/scenarios/4.5/4.5.pcap b/pcapminey/tests/scenarios/4.5/4.5.pcap new file mode 100644 index 0000000000000000000000000000000000000000..d2f5bc5e0015bb2e9e93785925f4486f21a093f6 Binary files /dev/null and b/pcapminey/tests/scenarios/4.5/4.5.pcap differ diff --git a/pcapminey/tests/scenarios/4.5/file.mp3 b/pcapminey/tests/scenarios/4.5/file.mp3 new file mode 100644 index 0000000000000000000000000000000000000000..084a7d1f8d13f74530095ab463c4b0b3bd809d97 Binary files /dev/null and b/pcapminey/tests/scenarios/4.5/file.mp3 differ diff --git a/pcapminey/tests/scenarios/4.5/sendfile.py b/pcapminey/tests/scenarios/4.5/sendfile.py new file mode 100644 index 0000000000000000000000000000000000000000..fdfd3eac54da306ff1677a402d6e403f78f1da27 --- /dev/null +++ b/pcapminey/tests/scenarios/4.5/sendfile.py @@ -0,0 +1,22 @@ +# -*- coding: utf8 -*- +__author__ = 'Viktor Winkelmann' + +from socket import * + +CHUNKSIZE = 1400 + +with open('file.mp3', 'rb') as inputfile: + s = socket(AF_INET, SOCK_DGRAM) + + data = inputfile.read(CHUNKSIZE) + while data != '': + s.sendto(data, ('192.168.123.37', 4242)) + data = inputfile.read(CHUNKSIZE) + + s.close() + + + + + + diff --git a/pcapminey/tests/scenarios/4.6/4.6.pcap b/pcapminey/tests/scenarios/4.6/4.6.pcap new file mode 100644 index 0000000000000000000000000000000000000000..774993a9f62a27e516fd882994133762be688bec Binary files /dev/null and b/pcapminey/tests/scenarios/4.6/4.6.pcap differ diff --git a/pcapminey/tests/scenarios/4.6/enc_file.py b/pcapminey/tests/scenarios/4.6/enc_file.py new file mode 100644 index 0000000000000000000000000000000000000000..f2970f32f02085c2df9bbd19022365e4cc797c1a --- /dev/null +++ b/pcapminey/tests/scenarios/4.6/enc_file.py @@ -0,0 +1,14 @@ +# -*- coding: utf8 -*- +__author__ = 'Viktor Winkelmann' + +from Crypto.Cipher import AES + +PASSWORD='0123456789ABCDEF' + +with open('orig_file.exe', 'rb') as input: + with open('file.aes', 'wb') as output: + aes = AES.new(PASSWORD, AES.MODE_CBC, '\x00'*16) + data = input.read() + while len(data) % 16 != 0: + data += '\x00' + output.write(aes.encrypt(data)) \ No newline at end of file diff --git a/pcapminey/tests/scenarios/4.6/file.aes b/pcapminey/tests/scenarios/4.6/file.aes new file mode 100644 index 0000000000000000000000000000000000000000..7400483b38a1f54a6e16a8ce01872bf0e39725a7 Binary files /dev/null and b/pcapminey/tests/scenarios/4.6/file.aes differ diff --git a/pcapminey/tests/scenarios/4.6/orig_file.exe b/pcapminey/tests/scenarios/4.6/orig_file.exe new file mode 100644 index 0000000000000000000000000000000000000000..52d6e60fb8ea786f9bcbfb8fb2b2b197fd1e26de Binary files /dev/null and b/pcapminey/tests/scenarios/4.6/orig_file.exe differ diff --git a/pcapminey/tests/scenarios/4.6/sendfile.py b/pcapminey/tests/scenarios/4.6/sendfile.py new file mode 100644 index 0000000000000000000000000000000000000000..f513c6f7d9ee324974801970c7b164766655b9b1 --- /dev/null +++ b/pcapminey/tests/scenarios/4.6/sendfile.py @@ -0,0 +1,15 @@ +# -*- coding: utf8 -*- +__author__ = 'Viktor Winkelmann' + +from socket import socket + +with open('file.aes', 'rb') as inputfile: + data = inputfile.read() + + h = 'this is the header!(§$%113550987' + t = 'TRAILER_' + + s = socket() + s.connect(('192.168.123.37', 4242)) + s.sendall(h + data + t) + s.close() diff --git a/pcapminey/tests/scenarios/testScenarios.py b/pcapminey/tests/scenarios/testScenarios.py new file mode 100644 index 0000000000000000000000000000000000000000..4ddb04bad06cfe6eedf622e0b51c98a0a7a5bed3 --- /dev/null +++ b/pcapminey/tests/scenarios/testScenarios.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python2.7 +# -*- coding: utf8 -*- +__author__ = 'Viktor Winkelmann' + +import sys + +sys.path.append('../..') +import unittest + +from core.Dispatcher import Dispatcher +import shutil +import os + + +class TestScenarios(unittest.TestCase): + OUTPUT_PATH = 'tmp_testscenarios' + + @classmethod + def findFile(cls, target): + foundFiles = [] + for root, dirs, files in os.walk(cls.OUTPUT_PATH): + for file in files: + with open(os.path.join(root, file), 'rb') as f: + foundFiles.append(f.read()) + + return target in foundFiles + + def setUp(self): + if os.path.exists(self.OUTPUT_PATH): + shutil.rmtree(self.OUTPUT_PATH) + + def tearDown(self): + if os.path.exists(self.OUTPUT_PATH): + shutil.rmtree(self.OUTPUT_PATH) + + def test_scenario4_1(self): + target = open('4.1/file.zip', 'rb').read() + + d = Dispatcher('4.1/4.1.pcap', self.OUTPUT_PATH, verifyChecksums=False) + d.run() + self.assertTrue(self.findFile(target)) + + def test_scenario4_2(self): + target = open('4.2/file.jpg', 'rb').read() + + d = Dispatcher('4.2/4.2.pcap', self.OUTPUT_PATH, verifyChecksums=False) + d.run() + self.assertTrue(self.findFile(target)) + + def test_scenario4_3(self): + target = open('4.3/file.pdf', 'rb').read() + + d = Dispatcher('4.3/4.3.pcap', self.OUTPUT_PATH, verifyChecksums=False) + d.run() + self.assertTrue(self.findFile(target)) + + def test_scenario4_4(self): + target = open('4.4/file.mp3', 'rb').read() + + d = Dispatcher('4.4/4.4.pcap', self.OUTPUT_PATH, verifyChecksums=False) + d.run() + self.assertTrue(self.findFile(target)) + + def test_scenario4_5(self): + target = open('4.5/file.mp3', 'rb').read() + + d = Dispatcher('4.5/4.5.pcap', self.OUTPUT_PATH, verifyChecksums=False) + d.run() + self.assertTrue(self.findFile(target)) + + + def test_scenario4_6(self): + target = open('4.6/file.aes', 'rb').read() + + d = Dispatcher('4.6/4.6.pcap', self.OUTPUT_PATH, True, verifyChecksums=False) + d.run() + + h = 'this is the header!(§$%113550987' + t = 'TRAILER_' + self.assertTrue(self.findFile(h + target + t)) + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/pcapminey/tests/webextract/web.pcap b/pcapminey/tests/webextract/web.pcap new file mode 100644 index 0000000000000000000000000000000000000000..6118b797c30eb30cf11516c0fade3926bb41aa21 Binary files /dev/null and b/pcapminey/tests/webextract/web.pcap differ diff --git a/pcapminey/tests/webextract/web_light.pcap b/pcapminey/tests/webextract/web_light.pcap new file mode 100644 index 0000000000000000000000000000000000000000..aa4b584f1033d6be55757331c4085327f97ff8ca Binary files /dev/null and b/pcapminey/tests/webextract/web_light.pcap differ diff --git a/pcapminey/tests/zipextract/testStreamBuilder.py b/pcapminey/tests/zipextract/testStreamBuilder.py new file mode 100644 index 0000000000000000000000000000000000000000..f48bdc313ddcbb62d2f8cf245c639e46e27168db --- /dev/null +++ b/pcapminey/tests/zipextract/testStreamBuilder.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python2.7 +# -*- coding: utf8 -*- +__author__ = 'Viktor Winkelmann' + +import sys +sys.path.append('../..') +import unittest + +from core.Streams.StreamBuilder import * + + +class TestZipExtract(unittest.TestCase): + def setUp(self): + inputfile = 'zipdownload.pcap' + self.zipfile = 'testfile.zip' + + self.builder = StreamBuilder(inputfile) + + self.zipheader = b'\x50\x4B\x03\x04' + + self.testfile = open(self.zipfile, 'rb') + + def tearDown(self): + self.testfile.close() + + def test_tcp_streams(self): + self.assertEqual(len(self.builder.tcpStreams), 2) + self.assertEqual(len(self.builder.tcpStreams[0]), 1) + self.assertEqual(len(self.builder.tcpStreams[1]), 1032) + + def test_extract_zip(self): + targetdata = self.testfile.read() + zipstream = self.builder.tcpStreams[1] + + zipindex = zipstream.getFirstBytes(1024).find(self.zipheader) + zipfile = zipstream.getAllBytes()[zipindex:] + self.assertEqual(zipindex, 276) + self.assertEqual(targetdata, zipfile) + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/pcapminey/tests/zipextract/testZipPlugin.py b/pcapminey/tests/zipextract/testZipPlugin.py new file mode 100644 index 0000000000000000000000000000000000000000..ee1781540a721a463130799b788fec2bb95b8a0f --- /dev/null +++ b/pcapminey/tests/zipextract/testZipPlugin.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python2.7 +# -*- coding: utf8 -*- +__author__ = 'Viktor Winkelmann' + +import sys + +sys.path.append('../..') +import unittest +from core.Streams.StreamBuilder import * +from core.Plugins.PluginManager import * + + +class TestZipExtract(unittest.TestCase): + def setUp(self): + inputfile = 'zipdownload.pcap' + self.zipfile = 'testfile.zip' + + self.builder = StreamBuilder(inputfile) + + self.pm = PluginManager() + self.zipplugin = self.pm.dataRecognizers["zip"] + + self.testfile = open(self.zipfile, 'rb') + + def tearDown(self): + self.testfile.close() + + def test_extract_zip(self): + targetdata = self.testfile.read() + streamdata = self.builder.tcpStreams[1].getAllBytes() + + zipindices = self.zipplugin.findNextOccurence(streamdata, 0, 1024) + zipfile = streamdata[zipindices[0]:] + self.assertEqual(targetdata, zipfile) + + +if __name__ == '__main__': + unittest.main() diff --git a/pcapminey/tests/zipextract/testfile.zip b/pcapminey/tests/zipextract/testfile.zip new file mode 100644 index 0000000000000000000000000000000000000000..ee8c6c3989775f5f4976811894911be3b7aa6867 Binary files /dev/null and b/pcapminey/tests/zipextract/testfile.zip differ diff --git a/pcapminey/tests/zipextract/zipdownload.pcap b/pcapminey/tests/zipextract/zipdownload.pcap new file mode 100644 index 0000000000000000000000000000000000000000..7bd181517240feb7d4deb6a32c1e09cc37f709d1 Binary files /dev/null and b/pcapminey/tests/zipextract/zipdownload.pcap differ diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..3365f6c2a9ecef040455491c45036f0863dcbb1a --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +cymruwhois==1.6 +dpkt==1.9.4 +regex==2020.11.13 +simplejson==3.17.2