% \documentclass[11pt,fleqn]{article} % Default font size and left-justified equations
\documentclass[11pt]{article} % Default font size and left-justified equations

%\usepackage{standalone}

% \usepackage{todonotes}
% use \todo{note} OR \missingfigure{Add my picture here}
\usepackage{longtable}
\include{structure} 

% slg commented out:
% \usepackage{arabtex}

\raggedbottom

\begin{document}

%define macros for commonly used terms that require special formatting
\newcommand \bulk {\textit{bulk\_extractor}\xspace}
\newcommand \beapi {\textit{be13\_api}\xspace}
\newcommand \viewer {\textbf{Bulk Extractor Viewer}\xspace}

\hypersetup{%
    pdfborder = {0 0 0}
}

\lstdefinestyle{customfile}{
basicstyle=\footnotesize\ttfamily, frame=single, float=htpb}

\input{./title.tex}

\pagenumbering{roman}
\setlength{\parindent}{0pt} %remove indenting from whole document
\newpage
\thispagestyle{empty}
\mbox{}
\newpage
\section*{One Page Quickstart for Linux \& Mac OS X Users}
This page provides a very brief introduction to downloading, installing and running \bulk. 
\begin{enumerate}
\item If you do not already have one, obtain a disk image on which to run \bulk. Sample images can be downloaded from \url{http://digitalcorpora.org/corpora/disk-images}. Suggestions include \texttt{nps-2009-domexusers} and \\ \texttt{nps-2009-ubnist1.gen3.E01}.
\item Download the latest version of \bulk. It can be obtained from \url{http://digitalcorpora.org/downloads/bulk_extractor/}. The file is called \texttt{bulk\_extractor-x.y.z.tar.gz} where x.y.z is the latest version. 

\item Un-tar and un-zip the file.  In the newly created \textit{bulk\_extractor-x.y} directory, run the following commands:

\begin{Verbatim}[commandchars=\\\{\}]
\verbbf{./configure}
\verbbf{make}
\verbbf{sudo make install}
\end{Verbatim}
[Refer to \textbf{\Autoref{InstallOnLinuxMac}} \textbf{\nameref{InstallOnLinuxMac}}. Note, for full functionality, some users may need to first download and install dependent library files. Instructions are outlined in the referenced section.]

\item To run \bulk from the command line, type the following command:
\begin{Verbatim}[commandchars=\\\{\}]
\verbbf{bulk_extractor -o output mydisk.raw}
\end{Verbatim} 

In the above command, \textbf{output} is the directory that will be created to store \bulk results.  It can not already exist. The input \textbf{mydisk.raw} is the disk image to be processed.
[See  \textbf{\Autoref{RunFromCommandLine}} \textbf{\nameref{RunFromCommandLine}}]

\item To run \bulk from the \viewer, navigate to the directory called \textit{/java\_gui} in the \bulk folder and run the following command:
\begin{Verbatim}[commandchars=\\\{\}]
\verbbf{./BEViewer}
\end{Verbatim}

In the \viewer, click on the Gear/down arrow icon as depicted below.
 
\noindent
\begin{minipage}{\linewidth}
	\makebox[\linewidth]{%
	\includegraphics[keepaspectratio=true,scale=0.6]{viewerPics/runBulkSnippet}}
\label{fig:runBulk1}
\end{minipage}

A window will pop up and the first two input boxes allow you to select an Image File and specify an Output Feature Directory to create. Enter both of those and then select the button at the bottom of the window titled "Start bulk\_extractor" to run \bulk.
[See \textbf{\Autoref{RunningViewer}} \textbf{\nameref{RunningViewer}}]

\item Whether \bulk was run from the command line or the \viewer tool, after the run the resulting output files will be contained in the specified output directory. Open that directory and verify files have been created. There should be 15-25 files. Some will be empty and others will be populated with data.

\item Users can join the google email users group for more information and help with any issues encountered. Email \textbf{bulk\_extractor-users+subscribe@googlegroups.com} with a blank message to join.
\end{enumerate}
\newpage

\section*{One Page Quickstart for Windows Users}
This page provides a very brief introduction to downloading, installing and running \bulk. 
\begin{enumerate}
\item If you do not already have one, obtain a disk image on which to run \bulk. Sample images can be downloaded from \url{http://digitalcorpora.org/corpora/disk-images}. Suggestions include \texttt{nps-2009-domexusers} and \\ \texttt{nps-2009-ubnist1.gen3.E01}.
\item Download the latest version of the \bulk Windows installer. It can be obtained from \url{http://digitalcorpora.org/downloads/bulk_extractor}. The file to download is called \texttt{bulk\_extractor-x.y.z-windowsinstaller.exe} where x.y.z is the latest version number.  Run the installer file. This will automatically install \bulk on your machine. The automatic installation includes the complete \bulk system as well as the \viewer tool. [See \textbf{\Autoref{InstallOnWindows}} \textbf{\nameref{InstallOnWindows}}]

\item To run \bulk from the command line, type the following command:
\begin{Verbatim}[commandchars=\\\{\}]
\verbbf{bulk_extractor -o output mydisk.raw}
\end{Verbatim} 

In the above command, \textbf{output} is the directory that will be created to store \bulk results.  It can not already exist. The input \textbf{mydisk.raw} is the disk image to be processed.
[See  \textbf{\Autoref{RunFromCommandLine}} \textbf{\nameref{RunFromCommandLine}}]

\item To run \bulk from the \viewer, run the program \textbf{Bulk Extractor X.Y} from the Start Menu. 

In the \viewer, click on the Gear/down arrow icon as depicted below. 
\noindent
\begin{minipage}{\linewidth}
	\makebox[\linewidth]{%
	\includegraphics[keepaspectratio=true,scale=0.6]{viewerPics/runBulkSnippet}}
\label{fig:runBulk2}
\end{minipage}

A window will pop up and the first two input boxes allow you to select an Image File and specify an Output Feature Directory to create. Enter both of those and then select the button at the bottom of the window titled "Start bulk\_extractor" to run \bulk.
[See \textbf{\Autoref{RunningViewer}} \textbf{\nameref{RunningViewer}}]

\item Whether \bulk was run from the command line or the \viewer tool, after the run the resulting output files will be contained in the specified output directory. Open that directory and verify files have been created. There should be 15-25 files. Some will be empty and others will be populated with data.

\item Users can join the google email users group for more information and help with any issues encountered. Email \textbf{bulk\_extractor-users+subscribe@googlegroups.com} with a blank message to join.
\end{enumerate}
\newpage


\tableofcontents
\newpage
\pagenumbering{arabic}





\newpage

\section{Introduction}

\subsection{Overview of \bulk}
\bulk is a program that extracts features such as email addresses, credit card numbers, URLs, and other types of information from digital evidence media. It is a useful forensic investigation tool for many tasks such as malware and intrusion investigations, identity investigations and cyber investigations, as well as analyzing imagery and password cracking. The program provides several unusual capabilities including:
\begin{itemize}
\item It finds email addresses, URLs and credit card numbers that other tools miss because it can process compressed data (like ZIP, PDF and GZIP files) and incomplete or partially corrupted data. It can carve JPEGs, office documents and other kinds of files out of fragments of compressed data. It will detect and carve encrypted RAR files.
\item It builds word lists based on all of the words found within the data, even those in compressed files that are in unallocated space. Those word lists can be useful for password cracking.
\item It is multi-threaded; running \bulk on a computer with twice the number of cores typically makes it complete a run in half the time.
\item It creates histograms showing the most common email addresses, URLs, domains, search terms and other kinds of information on the drive.
\end{itemize}

\bulk operates on disk images, files or a directory of files and extracts useful information without parsing the file system or file system structures. The input is split into pages and processed by one or more scanners. The results are stored in feature files that can be easily inspected, parsed, or processed with other automated tools. \bulk also creates histograms of features that it finds. This is useful because features such as email addresses and internet search terms that are more common tend to be important. \\  

In addition to the capabilities described above, \bulk also includes:
\begin{itemize}	
	\item A graphical user interface, \viewer, for browsing features stored in feature files and for launching \bulk scans
	\item A small number of python programs for performing additional analysis on feature files
\end{itemize} 
\bulk 1.5 detects and optimistically decompresses data in ZIP, GZIP, RAR, and Microsoft's Hibernation files. This has proven useful, for example, in recovering email addresses from fragments of compressed files found in unallocated space. \\

\bulk contains a simple but effective mechanism for protecting against decompression bombs. It also has capabilities specifically designed for Windows and malware analysis including decoders for the Windows PE, Linux ELF, VCARD, Base16, Base64 and Windows directory formats.\\

\bulk gets its speed through the use of compiled search expressions and multi-threading. The search expressions are written as pre-compiled regular expressions, essentially allowing \bulk to perform searches on disparate terms in parallel. Threading is accomplished through the use of an analysis thread pool.  After the features have been extracted, \bulk builds a histogram of email addresses, Google search terms, and other extracted features. Stop lists can also be used to remove features not relevant to a case. \\

\bulk is distinguished from other forensic tools by its speed and thoroughness. Because it ignores file system structure, \bulk can process different parts of the disk in parallel. This means that an 8-core machine will process a disk image roughly 8 times faster than a 1-core machine. \bulk is also thorough. It automatically detects, decompresses, and recursively re-processes data that has been compressed with a variety of algorithms. Our testing has shown there is a significant amount of compressed data in the unallocated regions of file systems missed by most forensics tools that are commonly in use today\cite{encodedEvidence}.  Another advantage of ignoring file systems is that \bulk can be used to process any kind of digital media. The program has been used to process hard drives, SSDs, optical media, camera cards, cell phones, network packet dumps, and other kinds of digital information.\\

Between 2005 and 2008, the \bulk team interviewed law enforcement regarding their use of forensic tools. Law enforcement officers wanted a highly automated tool for finding email addresses and credit card numbers (including track 2 information), phone numbers, GPS coordinates and EXIF information from JPEGs, search terms (extracted from URLs), and all words that were present on the disk (for password cracking). The tool needed to run on Windows, Linux and Mac OS X systems with no user interaction. It also had to operate on raw disk images, split-raw volumes and E01 files. The tool needed to run at the maximum I/O speed of the physical drive and never crash. Through these interviews, the initial  requirements for the \bulk system were developed. Over the past five years, we have worked to create the tool that those officers desired. \\

\subsubsection{A \bulk Success Story}
One early \bulk success story comes from the City of San Luis Obispo Police Department in the Spring of 2010. The District Attorney filed charges against two individuals for credit card fraud and possession of materials to commit credit card fraud. The defendants were arrested with a computer. Defense attorneys were expected to argue that the defendants were unsophisticated and lacked knowledge to commit the crime. The examiner was given a 250 GB drive the day before the preliminary hearing; typically it would take several days to conduct a proper forensic investigation of that much data.\\

\bulk found actionable evidence in only two and a half hours including the following information:
\begin{itemize}
\item There were over 10,000 credit card numbers on the hard drive (illegal materials).  Over 1000 of the credit card numbers were unique. 
\item The most common email address belonged to the primary defendant (evidence of possession). 
\item The most commonly occurring internet search engine queries concerned credit card fraud and bank identification numbers (evidence of intent). 
\item The most commonly visited websites were in a foreign country whose primary language is spoken by the defendant (evidence of flight risk). 
\end{itemize}
Armed with this data, the defendants were held without bail.\\

As \bulk has been deployed and used in different applications, it has evolved to meet additional requirements. This manual describes use cases for the \bulk system and demonstrates how users can take full advantage of all of its capabilities.

\subsection{Purpose of this Manual}
This User Manual is intended to be useful to new, intermediate and experienced users of \bulk. It provides an in-depth review of the functionality included in \bulk and shows how to access and utilize features through both command line operation and the \viewer. This manual includes working examples with links to the input data (disk images)  used, giving users the opportunity to work through the examples and utilize all aspects of the system. 

\subsection{Conventions Used in this Manual}
This manual uses standard formatting conventions to highlight file names, directory names and example commands. The conventions for those specific types are described in this section. \\

Names of programs including the post-processing tools native to \bulk and third-party tools are shown in \textbf{bold}, as in \textbf{tcpflow}.\\

File names are displayed in a fixed width font. They will appear as \texttt{filename.txt} within the text throughout the manual.\\

Directory names are displayed in italics. They appear as \textit{directoryname/} within the text. The only exception is for directory names that are part of an example command. Directory names referenced in example commands appear in the example command format.\\

Scanner names are denoted with bold, italicized text. They are always specified in lower-case, because that is how they are referred in the options and usage information for \bulk. Names will appear as \textbf{\textit{scannername}}.\\

This manual contains example commands that should be typed in by the user. A command entered at the terminal is shown like this: \begin{Verbatim}[commandchars=\\\{\}]
\verbbf{command}
\end{Verbatim}

The first character on the line is the terminal prompt, and should not be typed. The black square is used as the standard prompt in this manual, although the prompt shown on a users screen will vary according to the system they are using.\\


\section{How \bulk Works}
\bulk finds email addresses, URLs,  and CCNs that other tools miss. This is due in part to the fact that \bulk optimistically decompresses and re-analyzes all data (e.g. zip fragments, gzip browser cache runs). The decompression operates on incomplete and corrupted data until decompression fails. \bulk can also build word lists for password cracking\\

There are three phases of operation in \bulk:  feature extraction, histogram creation, post processing as shown in Figure ~\ref{fig:overviewPic}. The output feature files contain extracted data designed for easy processing by third party programs or use in spreadsheet tools. The \bulk histogram system automatically summarizes features.\\

\begin{figure}
	\center
	\includegraphics[scale=.60]{archPics/howitworks.pdf}
	\caption{Three Phases of \bulk Operation}
	\label{fig:overviewPic}
\end{figure}

Features files are written using the feature recording system. As features are discovered, they are sent to the feature recorder and recorded in the appropriate file. Multiple scanners might write to the same feature file. For example, the  \textbf{\textit{exif}} scanner searches the file formats used by digital cameras and finds GPS coordinates in images. Those findings are written to the output file \texttt{gps.txt} by the \textbf{\textit{gps}} feature recorder. A separate scanner, the  \textbf{\textit{gps}} scanner, searches Garmin Trackpoint data and also finds GPS coordinates and writes them to \texttt{gps.txt}.  It is worth noting that some scanners also find more than one type of feature and write to several feature files. For example, the  \textbf{\textit{email}} scanner looks for email addresses, domains, URLs and RFC822 headers and writes them to \texttt{email.txt}, \texttt{domain.txt}, \texttt{url.txt}, \texttt{rfc822.txt} and \texttt{ether.txt} respectively.\\

A feature file contains rows of features. Each row is typically comprised of an offset, a feature, and the feature in evidence context although scanners are free to store whatever information they wish. A few lines of an email feature file might look like the following:\\ 
\lstset{style=customfile}
\begin{lstlisting}
OFFSET      FEATURE	        FEATURE IN EVIDENCE CONTEXT
48198832  domexuser2@gmail.com 	__<name>domexuser2@gmail.com/Home
48200361  domexuser2@live.com 	__<name>domexuser2@live.com</name
48413823  siege@preoccupied.net	'Brien  <siege@preoccupied.net>_l
\end{lstlisting}

The types of features displayed in the feature file will vary depending on what type of feature is being stored. However, all feature files use the same format with each row corresponding to one found instance of a feature and three columns describing the related data (offset, feature, and feature in evidence context). \\

Histograms are a powerful tool for understanding certain kinds of evidence. A histogram of emails allows us to rapidly determine the drive's primary user, the user's organization, primary correspondents and other email addresses. The feature recording system automatically makes histograms as data are processed. When the scanner writes to the feature recording system, the relevant histograms are automatically updated. \\ 

A histogram file will, in general, look like the following file excerpt:
\lstset{style=customfile}
\begin{lstlisting}
n=875 mozilla@kewis.ch (utf16=3)
n=651 charlie@m57.biz (utf16=120)
n=605 ajbanck@planet.nl
...
n=288 mattwillis@gmail.com
n=281 garths@oeone.com
n=226 michael.buettner@sun.com (utf16=2)
n=225 bugzilla@babylonsounds.com
n=218 berend.cornelius@sun.com
n=210 ips@mail.ips.es
n=201 mschroeder@mozilla.x-home.org
n=186 pat@m57.biz (utf16=1)
\end{lstlisting}
Each line shows a feature and the number of times that feature was found by \bulk (the histogram indicates how many times the item was found coded as UTF-16). Features are stored in the file in order of occurrence with most frequent features appearing at the top of the file and least frequent displayed at the bottom. \\

\bulk has multiple scanners that extract features. Each scanner runs in an arbitrary order. Scanners can be enabled or disabled which can be useful for debugging and speed optimization. Some scanners are recursive and actually expand the data they are exploring, thereby creating more data that \bulk can analyze. These blocks are called sbufs. The "s" stands for the word safe. All access to data in the sbuf is bounds-checked, so buffer overflow events are very unlikely. The sbuf data structure is one of the reasons that \bulk is so crash resistant. Recursion is used for, among other things, decompressing ZLIB and Windows HIBERFILE, extracting text from PDFs and handling compressed browser cache data. \\

The recursion process requires a new way to describe offsets. To do this, \bulk introduces the concept of the ``forensic path.'' The forensic path is a description of the origination of a piece of data. It might come from, for example, a flat file, a data stream, or a decompression of some type of data. Consider an HTTP stream that contains a GZIP-compressed email as shown in Figure ~\ref{fig:forensicPath}. A series of scanners will first find the ZLIB compressed regions in the HTTP stream that contain the email, decompress them, and then find the features in that email which may include email addresses, names and phone numbers. Using this method, \bulk can find email addresses in compressed data. The forensic path for the email addresses found indicate that it originated in an email, that was GZIP compressed and found in an HTTP stream. The forensic path of the email addresses features found might be represented as follows:
\lstset{style=customfile}
\begin{lstlisting}
11052168704-GZIP-3437  live.com  eMn='domexuser@live.com';var  srf_sDispM
11052168704-GZIP-3475  live.com  pMn='domexuser@live.com';var  srf_sDreCk
11052168704-GZIP-3512  live.com  eCk='domexuser@live.com';var  srf_sFT='<
\end{lstlisting}


\begin{figure}
	\center
	\includegraphics[scale=.60]{otherPics/forensicPath.jpg}
	\caption{Forensic path of features found in email lead back to HTTP Stream}
	\label{fig:forensicPath}
\end{figure}


The full functionality of \bulk is provided both through command line operation and the GUI tool, \viewer.  Both modes of operation work for Linux, Mac and Windows.  The following section describes how to download, install and run \bulk using either the command line or the \viewer.

\section{Running \bulk}
\bulk is a command line tool with an accompanying graphical user interface tool, \viewer. All of the command line functionality of \bulk is also available in the \viewer. Users can access the functionality in whichever way they prefer. In this manual we review the \bulk user options in both formats.\\

\bulk can be run on a Linux, Mac OS X or Windows system. The fastest way to run \bulk is on a Linux system. Running \bulk on Windows provides the same results, but the run will typically take 40 percent longer on the same hardware. The software can actually run faster on a Linux virtual machine running on Windows with VMware workstation than on the native Windows OS. 


\subsection{Installation Guide}
Installation instructions vary for Linux, Mac OS X users and Windows users. The following sections explain how to install \bulk.

\subsubsection{Installing on Linux or Mac OS X}
\label{InstallOnLinuxMac}
Before compiling \bulk for your platform, you may need to install other packages on your system which \bulk requires to compile cleanly and with a full set of capabilities.\\

\textbf{Dependencies for Fedora}\\
The following commands should add the appropriate packages:
\begin{Verbatim}[commandchars=\\\{\}]
\verbbf{sudo yum update}
\verbbf{sudo yum groupinstall development-tools}
\verbbf{sudo yum install flex}
\end{Verbatim}

\textbf{Dependencies for Debian (wheezy) or Ubuntu (13.0)}\\
The following command should add the appropriate libraries:
\begin{Verbatim}[commandchars=\\\{\}]
\verbbf{sudo apt-get -y install gcc g++ flex libewf-dev} 
\end{Verbatim}

\textbf{Dependencies for Mac OS X}\\
Mac OS X users must first install Apple's Xcode application (available in the OS X App store), and then install the command line tools. 
To install the command line tools in Mavericks and Yosemite, enter this command in the terminal:
\begin{Verbatim}[commandchars=\\\{\}]
\verbbf{xcode-select --install}
\end{Verbatim}

Other components can be downloaded using the MacPorts system. To install MacPorts, get the latest ports for your version of OS X here:
http://macports.com
After the latest ports are installed, you still need to make sure some optional packages are added using these commands:

\begin{Verbatim}[commandchars=\\\{\}]
\verbbf{sudo port install flex autoconf automake}
\verbbf{sudo port install libewf-devel}
\end{Verbatim}

Mac OS X users should note that libewf-devel may not be available in ports, and at present, libewf isn't new enough.  If the required version isn't available as a port, then download and un-tar the libewf source (for example, in /tmp), cd into the source directory and run:
\begin{Verbatim}[commandchars=\\\{\}]
\verbbf{./configure} 
\verbbf{make}
\verbbf{sudo make install}
\end{Verbatim}

\textbf{Download and Install \bulk}\\
Next, download the latest version of \bulk. The software can be downloaded from \url{http://digitalcorpora.org/downloads/bulk_extractor/}. The file to download will be called \texttt{bulk\_extractor-x.y.z.tar.gz} where x.y.z is the latest version. As of publication of this manual, the latest version of \bulk is 1.5. \\

After downloading the tar.gz file, decompress and un-tar it. Then, cd into the newly created \textit{bulk\_extractor-x.y.z} directory, and run the following commands to install \bulk in \textit{/usr/local/bin} (by default):

\begin{Verbatim}[commandchars=\\\{\}]
\verbbf{./configure}
\verbbf{make}
\verbbf{sudo make install}
\end{Verbatim}

With these instructions, the following directory will not be installed:
\begin{itemize}
\item{\textit{plugins/} - This is for C/C++ developers only. You can develop your own \bulk plugins which will then be run at run-time with the \bulk executable. Refer to the \textbf{\bulk Programmers Manual for Developing Scanner Plug-ins \cite{programmersmanual}} for more information.}
\end{itemize}
Instructions on running \bulk from the command line can be found in \textbf{\Autoref{RunFromCommandLine}}.\\
	
The \viewer tool is installed as part of the above installation process. Specific instructions on running it can be found in \textbf{\Autoref{RunningViewer}}.

\subsubsection{Installing on Windows}
\label{InstallOnWindows}
Windows users should download the Windows Installer for \bulk.  The file to download is located at \url{http://digitalcorpora.org/downloads/bulk_extractor/} and is called \texttt{bulk\_extractor-x.y.z-windowsinstaller.exe} where x.y.z is the latest version number (1.5.0 as of publication of this manual).   \\

Next, run the \texttt{bulk\_extractor-x.y.z-windowsinstaller.exe} file. This will automatically install \bulk on your machine. Because this file is not used by many Windows users, some anti-virus systems will try to manual delete it on download or block the download as shown in Figure ~\ref{fig:installer_download}.  Be aware that you may have to work around your anti-virus system. Additionally, some Windows versions will try to prevent you from running it. Figure ~\ref{fig:windowsWarning} shows the message Windows 8 displays when trying to run the installer. To run anyway, click on ``More info'' and then select ``Run Anyway.''\\
\begin{figure}
	\center
	\includegraphics[scale=.60]{installPics/installer_download.pdf}
	\caption{Anti-virus software, such as Symantec, often tries to block download of the installer file}
	\label{fig:installer_download}
\end{figure}

\begin{figure}
	\center
	\includegraphics[scale=.50]{installPics/windowsWarning2.pdf}
	\caption{Windows 8 warning when trying to run the installer}
	\label{fig:windowsWarning}
\end{figure}

When the installer file is executed, the installation will begin and show a dialog like the one shown in Figure ~\ref{fig:installer1}.  Users should select the default configuration, which will be the 64-bit configuration for 64-bit Windows systems, or the 32-bit configuration for 32-bit Windows systems.  Click on ``Install'' and the installer will install \bulk on your system and then notify you when it is complete.\\
\begin{figure}
	\center
	\includegraphics[scale=.70]{installPics/installer1.png}
	\caption{Dialog appears when the user executes the Windows Installer}
	\label{fig:installer1}
\end{figure}


The automatic installation includes the \viewer tool as well as the complete \bulk system that can be run from the command line. Java 6 or above must  be installed on the machine for the \viewer to run. Instructions on running \bulk from the command line can be found in  \textbf{\Autoref{RunFromCommandLine}}. Instructions on running it from the \viewer are located in \textbf{\Autoref{RunningViewer}}. \\

\subsection {Run \bulk from the Command Line}
\label{RunFromCommandLine}
The two main parameters required to run \bulk are an output directory and a disk image. The output directory must be a directory that does not already exist. The disk image can be either a file such as a disk image or a directory of individual files. Note that \bulk cannot process a directory of disk images.\\

In the following instructions, \textit{output} is the name of the directory that will be created to store the \bulk output. The file \texttt{mydisk.raw} is the name of the disk image that will be extracted by \bulk. \\

To run \bulk from the command line on any machine, type the following command:
\begin{Verbatim}[commandchars=\\\{\}]
\verbbf{bulk_extractor -o output mydisk.raw}
\end{Verbatim}

The above command on any of the supported operating systems assumes that the disk image \textit{mydisk.raw} is located in the directory where the command is being executed. However, you can point \bulk to a disk image found elsewhere on your machine by explicitly entering the path to that image.\\

The following text shows the output that is produced when \bulk is run on the file \texttt{nps-2010-emails.E01}. The information printed indicates the version number, input file, output directory and disk size. The screen is updated as \bulk runs  with status information. \bulk then prints performance information and the number of features found when the run is complete.

\begingroup
\footnotesize
\texttt{C:\textbackslash \textgreater \textbf{bulk\_extractor -o bulk\_extractor\textbackslash Output\textbackslash nps-2010-emails bulk\_extractor\textbackslash In\\putData\textbackslash nps-2010-emails.E01}}
\endgroup
\begingroup
\footnotesize
\begin{Verbatim}[fontfamily=courier]
bulk_extractor version: 1.5.0
Input file: bulk_extractor\InputData\nps-2010-emails.E01
Output directory: bulk_extractor\Output\nps-2010-emails
Disk Size: 10485760
Threads: 4
All data are read; waiting for threads to finish...
Time elapsed waiting for 1 thread to finish:
     (timeout in 60 min .)
Time elapsed waiting for 1 thread to finish:
    6 sec (timeout in 59 min 54 sec.)
Thread 0: Processing 0

All Threads Finished!
Producer time spent waiting: 0 sec.
Average consumer time spent waiting: 8.32332 sec.
Phase 2. Shutting down scanners
Phase 3. Creating Histograms
   ccn histogram...   ccn_track2 histogram...   domain histogram...
   email histogram...   ether histogram...   find histogram...
   ip histogram...   lightgrep histogram...   tcp histogram...
   telephone histogram...   url histogram...   url microsoft-live...
   url services...   url facebook-address...   url facebook-id...
   url searches...Elapsed time: 11.1603 sec.
Overall performance: 0.939557 MBytes/sec
Total email features found: 67
\end{Verbatim}
\endgroup

Note that \bulk automatically choose to use 4 threads because the program was run on a computer with 4 cores. In general, \bulk automatically determines the number of cores to use. Therefore, it is not necessary to set the number of threads unless you want to limit the number to use.\\

After running \bulk, examine the output directory specified by name in the run command. There should now be a number of generated output files in that directory. There are several categories of output created for each \bulk run. First, there are feature files grouped by category, which contain the features found and include the path, feature and context. Second, there are histogram files that allow users to quickly see the features grouped by the frequency in which they occur. Certain kinds of files, such as JPEGs and KML files, may be carved into directories. Finally, \bulk creates a file \texttt{report.xml}, in DFXML format, that captures the provenance of the run. After \bulk has been run, all of these files will be found in the output directory specified by the user.\\

The text below shows the results of running the command \textbf{ls -s} within the output directory from the \bulk run on the disk image \texttt{nps-2010-emails.E01}. The numbers next to the file names indicate the file size and show that several of the files, including \texttt{email.txt} and \texttt{domain.txt}, were populated with features during the run.

\begingroup
\footnotesize
\texttt{C:\textbackslash bulk\_extractor\textbackslash Output\textbackslash nps-2010-emails\textgreater \textbf{ls -s}}
\endgroup
\begingroup
\footnotesize
\begin{Verbatim}[fontfamily=courier]
total 303
 0 aes_keys.txt               0 kml.txt
 0 alerts.txt                 0 lightgrep.txt
 0 ccn.txt                    0 lightgrep_histogram.txt
 0 ccn_histogram.txt          0 rar.txt
 0 ccn_track2.txt             8 report.xml
 0 ccn_track2_histogram.txt   0 rfc822.txt
64 domain.txt                 0 tcp.txt
 1 domain_histogram.txt       0 tcp_histogram.txt
 0 elf.txt                    0 telephone.txt
16 email.txt                  0 telephone_histogram.txt
 4 email_histogram.txt       96 url.txt
 0 ether.txt                  0 url_facebook-address.txt
 0 ether_histogram.txt        0 url_facebook-id.txt
 1 exif.txt                   4 url_histogram.txt
 0 find.txt                   0 url_microsoft-live.txt
 0 find_histogram.txt         0 url_searches.txt
 0 gps.txt                    1 url_services.txt
 0 hex.txt                    0 vcard.txt
 0 ip.txt                    12 windirs.txt
 0 ip_histogram.txt           0 winpe.txt
 0 jpeg                       0 winprefetch.txt
 8 jpeg_carved.txt            88 zip.txt
 0 json.txt
\end{Verbatim}
\endgroup


There are numerous feature files produced by \bulk for each run. A feature file is a tab-delimited file that show a feature on each row. Each row includes a path, a feature and the context. The files are in UTF-8 format.\\

Any of the feature files created by \bulk may have an accompanying \texttt{*\_stopped.txt} file found in the output directory. This file will show all stopped entries of that type that have been found so that users can examine those files to make sure nothing critical has been hidden. A stopped features is a feature that appears in a stop list. The stop list is a list of features that are not of concern for a particular investigation. For example, users may input a stop list file to \bulk that contains numerous email addresses that should be ignored and not marked as a found feature. Rather than throwing away those results when they are found, \bulk will create a file, named \texttt{email\_stopped.txt} that shows all email addresses from the stop list that were found during the run. The stopped email addresses will not show up in the \texttt{email.txt} file. More information on creating and using stop lists can be found in \textbf{\Autoref{StoppedLists}}.\\

While the above commands are all that is required for basic operation, there are numerous usage options that allow the user to affect input and output, tuning, path processing mode, debugging, and control of scanners. All of those options are described when \bulk is run with the -h (help) option. It is important to note that the overwhelming tendency of users is to use many of these options; \textbf{however}, that is not generally recommended. Most of the time, the best way to run \bulk is with no options specified other than -o to specify the output directory. For best performance and results users should avoid adding them in general. Only advanced sers in specific cases should use these options.\\

Running \bulk with only the -h option specified produces the output shown in \textbf{\Autoref{HelpOutput}}.  To run any optional usage options, they should be inserted before the input and output options are specified. Specifically, the order should look like the following:

\begin{Verbatim}[commandchars=\\\{\}]
\verbbf{bulk_extractor [Usage Options] -o output mydisk.raw}
\end{Verbatim}

The specific order in which multiple usage options are specified matters. Some of the options are discussed within the following sections for specific use cases, other options are for programmer or experimental use. In general, avoid using the options unless indicated for a specific purpose.\\

\subsection{Run \bulk from \viewer}
\label{RunningViewer}

On a Linux or Mac OS X system, go to the directory where the \viewer is installed or specify the full path name to the jar file. It will be in the location where the \bulk code was installed and in the sub-directory labeled \textit{java\_gui}. From that directory, run the following command to start the \viewer:
\begin{Verbatim}[commandchars=\\\{\}]
\verbbf{./BEViewer}
\end{Verbatim}

\subsection {Run \bulk from Bulk Extractor Viewer}

Windows users should go to the Start menu and choose Programs->Bulk\_Extractor x.y.z->BE Viewer with Bulk\_extractor x.y.z (64-bit). If the 64-bit version can not be run on your machine, you can choose the 32-bit version. The Troubleshooting section describes some limits users of the 32-bit version might encounter.\\

When the \viewer starts up, it will look like Figure ~\ref{fig:startup}. The look and feel may vary slightly according to the specific operating system but all options should appear similar. To run \bulk from the viewer, click on the icon that looks like a gear with a down arrow. It is next to the Print icon below the Tools menu. Clicking on this icon will bring up the ``Run bulk\_extractor'' Window as shown in Figure ~\ref{fig:runBulk3}.\\
\begin{figure}
	\center
	\includegraphics[scale=.8]{viewerPics/startup.png}
	\caption{What \viewer looks like when it is started}
	\label{fig:startup}
\end{figure}


\begin{figure}
	\center
	\includegraphics{viewerPics/runBulk.png}
	%\captionsetup{font=LARGE}
	\caption{Clicking on the gear icon brings up this ``Run bulk\_extractor'' Window}
	\label{fig:runBulk3}
\end{figure}

Next, in the ``Run bulk\_extractor'' window select the Image File and Output Feature Directory to run \bulk. Figure ~\ref{fig:selectOutputDirectory} shows an example where the user has selected the file \texttt{nps-2010-emails.E01} as input and is going to create a directory called \textit{nps-2010-charlie-output} in the parent directory \textit{C:\textbackslash bulk\_extractor\textbackslash Output}. Note that figures may vary slightly in future versions of \bulk but the major functionality will remain the same.\\
\begin{figure}
    \includegraphics[scale=.8]{viewerPics/selectOutputDirectory.png}
	\caption{After selecting an Image File for input, the user must select an output directory to create}
	\label{fig:selectOutputDirectory}
\end{figure}
 
After selecting the input and output directories, click on the button at the bottom of the ``Run bulk\_extractor'' window labeled ``Start bulk\_extractor.'' This will bring up the window shown in Figure ~\ref{fig:runCompleteStatus} that updates as \bulk is running, providing status information during the run and after the run is complete. \\
\begin{figure}
	\includegraphics{viewerPics/runCompleteStatus.png}
	\caption{Status window that shows what happens as \bulk runs and indicates when \bulk is complete}
	\label{fig:runCompleteStatus}
\end{figure}

When the run is complete, a dialog will pop-up indicating the results are ready to be viewed. Figure ~\ref{fig:reportIsReady} shows this dialog. Click the ``Ok'' button which will return you to the main \viewer window to view the results of the run. The ``Reports'' window on the left will now show the newly created report. In this example, the report is called ``nps-2010-emails-output.'' Clicking once on this report name will expand the report and show all of the files that have been created as shown in Figure ~\ref{fig:listOfOutput}. \\
\begingroup
\center
\begin{figure}
	\includegraphics{viewerPics/reportIsReady.png}
	\caption{Dialog indicating the run of \bulk is complete and results are ready to be viewed}
	\label{fig:reportIsReady}
\end{figure}
\endgroup
\begin{figure}
	\includegraphics[scale=.8]{viewerPics/listOfOutput.png}
	\caption{Reports window shows the newly created report and all of the files created in that report}
	\label{fig:listOfOutput}
\end{figure}
Clicking on one of the files will bring that file up in the ``Feature File'' window in the middle of the screen. In the example, the user clicked on \texttt{email.txt} to view the email feature file. Clicking on one of the features, in this case \textbf{rtf\_text@textedit.com}, shows the feature in context within the feature file on the right-hand side of the window as shown in Figure ~\ref{fig:emailFeatureFile}.\\

\begin{figure}
	\includegraphics[scale=.7]{viewerPics/emailFeatureFile.png}
	\caption{While viewing the feature file, the user can select a feature to view with it full context in the feature file as shown in the right-hand side of the window}
	\label{fig:emailFeatureFile}
\end{figure}

The user can also view histogram files in the \viewer. Clicking on the file, \texttt{email\_histogram.txt} in the Reports window on the left hand side will bring up the contents of the histogram file in the middle window. It will also display the referenced feature file in the window below the histogram file. In this case, the referenced feature file is \texttt{email.txt}. Clicking on a feature in the histogram, in this example \textbf{rtf\_text@textedit.com}, will display the feature in context as found within the feature file on the right-hand side of the screen as shown in Figure ~\ref{fig:emailHistogramView}.\\

\begin{figure}
	\includegraphics[scale=.70]{viewerPics/emailHistogramView.png}
	\caption{User can view histograms of features, referenced feature files and specific features in context}
	\label{fig:emailHistogramView}
\end{figure}


\section{Processing Data}
\subsection{Types of Input Data}

The \bulk system can handle multiple image formats including E01, raw, split raw and individual disk files as well as raw devices or files.  It can also operate on memory and packet captures, although packet captures will be more completely extracted if you pre-process them with \textbf{tcpflow}.\\

The scanners all serve different functions and look for different types of information. Often, a feature will be stored in a format not easily accessible and will require multiple scanners to extract the feature data. For example, some PDF files contain text data but the PDF format is not directly searchable by the scanner that finds email addresses or the scanner that looks for keywords. \bulk resolves this by having the two scanners work together. The \textbf{\textit{pdf}} scanner will first extract all of the text from the PDF and then the other scanners will look at the extracted text for features. This is important to remember when turning scanners off and on, as scanners work together to retrieve the features from the disk image. The types of information examined, extracted or carved by the existing \bulk scanners are as described in Table ~\ref{tab:inputdata}, along with the scanners that process them and the specific sections where they are referenced in this manual.

\begin{longtable}{|p{2 cm}|p{6cm}|p{3 cm}|}
\caption{Input Data Processed by the Scanners}\\
\hline
\textbf{Scanner Name} & \textbf{Data Type} & \textbf{Section Discussed in Manual} \\
\endfirsthead
\hline
\textbf{Scanner Name} & \textbf{Data Type} & \textbf{Section Discussed in Manual} \\
\endhead
\hline
\endfoot
\hline
\endlastfoot

\hline\textbf{\textit{accts}} & Numeric accounts, such as phone numbers and CCNs\\
\hline\textbf{\textit{aes}} & In-memory AES keys from their key schedules & \Autoref{cyber}\\
\hline\textbf{\textit{base16}} & Base 16 (hex) encoded data (includes MD5 codes embedded in the data) & \Autoref{cyber}\\
\hline\textbf{\textit{base64}} & Base 64 code & \Autoref{compressedProcessing} and \Autoref{cyber} \\
\hline\textbf{\textit{elf}} & Executable and Linkable Format (ELF) & \Autoref{malware} \\
\hline\textbf{\textit{exif}} & EXIF structures from JPEGS (and carving of JPEG files) & \Autoref{imagery} \\
\hline\textbf{\textit{facebook}} & Facebook HTML & \\
\hline\textbf{\textit{gps}} & XML from Garmin GPS devices (processed) & \Autoref{identity}\\
\hline\textbf{\textit{gzip}} & GZIP files and ZLIB-compressed GZIP streams & \Autoref{compressedProcessing} and \Autoref{cyber} \\
\hline\textbf{\textit{hashdb}} & NPS Hash Database support\\
\hline\textbf{\textit{hiber}} & Windows Hibernation File Fragments (decompressed and processed, not carved) & \Autoref{compressedProcessing}  \\
\hline\textbf{\textit{httplogs}} & HTTP log files\\
\hline\textbf{\textit{jpeg}} & JPEG carving. Default is only encoded JPEGs are carved. JPEGs without EXIFs are also carved & \Autoref{carving} and \Autoref{imagery}\\
\hline\textbf{\textit{json}} & JavaScript Object Notation files and objects downloaded from web servers, as well as JSON-like objects found in source code & \Autoref{malware}\\
\hline\textbf{\textit{kml}} & KML files (carved) & \Autoref{identity}\\
\hline\textbf{\textit{outlook}} & Outlook Compressable Encryption\\
\hline\textbf{\textit{pdf}} & Text from PDF files (extracted for processing not carved) & \Autoref{compressedProcessing} \\
\hline\textbf{\textit{rar}} &  RAR components in unencrypted archives are decrypted and processed. Encrypted RAR file are carved. & \Autoref{carving} \\
\hline\textbf{\textit{sqlite }} & SQLite3 database file detection and carving\\
\hline\textbf{\textit{vcard}} & vCard files (carved) & \Autoref{identity}\\
\hline\textbf{\textit{windirs}} & Windows FAT32 and NTFS directory entries& \Autoref{cyber}\\
\hline\textbf{\textit{winlnk}} & Windows LNK file carving and decoding\\
\hline\textbf{\textit{winpe}} & Windows Portable Executable (PE) (.exe and .dll files notated with MD5 hash of first 4k) & \Autoref{malware}\\
\hline\textbf{\textit{winprefetch}} & Windows Prefetch files, file fragments (processed) & \Autoref{malware} \\
\hline\textbf{\textit{zip}} & ZIP files and zlib streams (processed, and optionally carved) & \Autoref{carving} and \Autoref{compressedProcessing}\\
\hline
\end{longtable}
\label{tab:inputdata}

\subsection{Scanners}
\label{Scanners}
There are multiple scanners deployed with the \bulk system. For a detailed list of the scanners installed with your version of \bulk, run the following command:
\begin{Verbatim}[commandchars=\\\{\}]
\verbbf{bulk_extractor -H}
\end{Verbatim}

This command will show all of the scanners installed with additional information included about each scanner. Specifically, there is a description for each scanner, a list of the features it finds and any relevant flags. A sample of the output is below: 
\begingroup
\footnotesize
\begin{Verbatim}[fontfamily=courier]
Scanner Name: accts
flags:  NONE
Scanner Interface version: 3
Author: Simson L. Garfinkel
Description: scans for CCNs, track 2, and phone #s
Scanner Version: 1.0
Feature Names: alerts ccn ccn_track2 telephone

Scanner Name: base16
flags:  SCANNER_RECURSE
Scanner Interface version: 3
Author: Simson L. Garfinkel
Description: Base16 (hex) scanner
Scanner Version: 1.0
Feature Names: hex

...

Scanner Name: wordlist
flags:  SCANNER_DISABLED
Scanner Interface version: 3
Author:
Description:
Scanner Version:
Feature Names: wordlist
\end{Verbatim}
\endgroup

This output shows that the  \textbf{\textit{accts}} scanner looks for credit card numbers, credit card track 2 information and phone numbers and finds the feature names alerts, ccn, ccn\_track2 and telephone. This means it writes to the feature files \texttt{alerts.txt}, \texttt{ccn.txt}, \texttt{ccn\_track2.txt}, and \texttt{telephone.txt}.\\

The output also shows that the  \textbf{\textit{base16}} scanner is a recursive scanner (indicated by the flag SCANNER\_RECURSE) meaning it expands data or finds new data for other scanners to process. It also writes to the file \texttt{hex.txt}.\\

Finally, the output shows that the  \textbf{\textit{wordlist}} scanner is disabled by default (indicated by the flag SCANNER\_DISABLED). This means that if the user would like to use the  \textbf{\textit{wordlist}} scanner, it will have to be specifically enabled. The wordlist scanner is useful for password cracking and is discussed in \textbf{\Autoref{PasswordCracking}}. \\

In general, most users will not need to enable or disable scanners. The default settings installed with the \bulk system work best for the majority of users. However, individual scanners can be enabled or disabled for different purposes. To enable the  \textbf{\textit{wordlist}} scanner, which is disabled by default, use the following command:
\begin{Verbatim}[commandchars=\\\{\}]
\verbbf{bulk_extractor -e wordlist -o output diskimage.raw}
\end{Verbatim} 

Additionally, users can disable a scanner that is enabled by default. Most of the scanners are enabled by default. To disable the  \textbf{\textit{accts}} scanner, which is very CPU intensive, run the following command:
\begin{Verbatim}[commandchars=\\\{\}]
\verbbf{bulk_extractor -x accts -o output diskimage.raw} 
\end{Verbatim}

The command -E disables all scanners, then enables the one that follows the option. For example, to disable all scanners except the \textbf{\textit{aes}} scanner, use the following command:
\begin{Verbatim}[commandchars=\\\{\}]
\verbbf{bulk_extractor -E aes -o output diskimage.raw}
\end{Verbatim} 

The options -E, -e and -x are all processed in order. So, the following command will also disable all scanners and then enable the \textbf{\textit{aes}} scanner: 
\begin{Verbatim}[commandchars=\\\{\}]
\verbbf{bulk_extractor -x all -e aes -o output diskimage.raw}
\end{Verbatim} 

Some of the scanners installed with \bulk have parameters that can be set and utilized by advanced users for different purposes. Those parameters are also described in the -H output described above (as well as the -h output) and include the following:
\begingroup
\footnotesize
{
\fontfamily{courier}\selectfont
\verbatiminput{SettableOptions.txt}
}
\endgroup

To use any of these options, the user should specify the -S with the name=value pair when running \bulk as in the following example:
\begin{Verbatim}[commandchars=\\\{\}]
\verbbf{bulk_extractor -S name=value -o output diskimage.raw}
\end{Verbatim} 
As with the other scanner and \bulk usage options, most users will not have to use any of these options. 

\subsection{Carving}
\label{carving}

File carving is a special kind of carving in which files are recovered. File carving is useful for both data recovery and forensic investigations because it can recover files when sectors containing file system metadata are either overwritten or damaged \cite{digitalmediatriage}. Currently, \bulk provides carving of contiguous JPEG, ZIP and RAR files. To carve fragmented files we recommend \textbf{PhotoRec} (free) or \textbf{Adroit Photo Recovery} (commercial). Additionally, \textbf{Forensics Toolkit} and \textbf{EnCase Forensic} provide some carving capability on fragmented files.\\

Carved results are stored in two different places. First, a file listing all the files that are carved are written to a corresponding .txt file: JPEG files to \texttt{jpeg\_carved.txt}, ZIP files to \texttt{unzip.txt} and RAR files to \texttt{unrar.txt}. Second, the carved JPEG, ZIP and RAR files are placed in binned directories that are named \textit{/jpeg}, \textit{/unzip} and \textit{/unrar} respectively. For example, all carved JPEGs will go in the directory \textit{/jpeg}. The output files are further binned with 1000 files in each directory. The directory names are 3 decimal digits. If there are more than 999,000 carved files of one type, then the next set of directories are named with 4 digits. File names for JPEGs are the forensicpath.jpg. File names for the ZIP carver are the forensicpath\_filename. If the ZIP file name has slashes in it (denoting directories), they are turned into '\_' (underbars). For example, the file \texttt{mydocs/output/specialfile} will be named \texttt{mydocs\_output\_specialfile}. \\

\begin{table}[!ht]
\centering
\caption{There are three carving modes in \bulk that are specified separately for each file type, JPEG, ZIP and RAR.}
\label{tab:encodings}
\begin{tabular}{|m{2 cm}|m{7 cm}|}
\hline \hline
\textbf{Mode} & \textbf{Mode Description} \\
\hline
0 & Do not carve files of the specified type.\\
\hline
1 & Only carve encoded files of the specified type \\
\hline
2 & Carve everything of the specified type.\\
\hline
\end{tabular}
\end{table}

As the above table describes, there are three carving modes in \bulk that can be specified separately for each file type, JPEG, ZIP or RAR. The first mode, mode 0, explicitly tells \bulk not to carve files of that type. The second mode, mode 1, is on by default and tells \bulk to carve only encoded files of that type. If the user is running the ZIP carver in mode 1 and there is a simple ZIP file, it will not get carved. However, if there is an encoded attachment of that file (like Base64) it will get carved. The final mode, mode 2, will carve everything of that type. There is no way to specify which types of files (particular extensions) will get carved and which will not in mode 2. For example, \bulk will carve both JPEGs and doc files. It carves whatever is encountered.\\

To specify the carving modes for \bulk, command line arguments can be specified. To modify the JPEG carving modes, type the following where carve mode 1=default value that does not need to be specified (carve encoded), 0=no carving or 2=carve everything:
\begin{Verbatim}[commandchars=\\\{\}]
\verbbf{bulk_extractor -S jpeg_carve_mode=1 -o output diskimage.raw}
\end{Verbatim} 
To modify the ZIP carving modes, type the following  where carve mode 1=default value that does not need to be specified (carve encoded), 0=no carving or 2=carve everything:
\begin{Verbatim}[commandchars=\\\{\}]
\verbbf{bulk_extractor -S unzip_carve_mode=1 -o output diskimage.raw}
\end{Verbatim} 
To modify the RAR carving modes, type the following  where carve mode 1=default value that does not need to be specified (carve encoded), 0=no carving or 2=carve everything:
\begin{Verbatim}[commandchars=\\\{\}]
\verbbf{bulk_extractor -S unrar_carve_mode=1 -o output diskimage.raw}
\end{Verbatim} 
Any combination of the carving mode options can be specified for a given run. The carvers can run in any combination of modes. For example, the JPEG carver can be run in mode 2 while the RAR carving is turned off in mode 1 and the ZIP carver carves only encoded files in mode 1.\\

Because \bulk can carve files and preserve original file extensions, there is a real possibility that \bulk might be carving out malware. There is no protection in \bulk against putting malware in a file on your hard drive. Users running \bulk to look for malware should turn off all anti-virus software because the anti-virus program will think its creating malware and stop it. Then the user should carefully scan the results looking for malware before re-enabling the anti-virus. \\


\subsection{Suppressing False Positives}
\label{StoppedLists}
Modern operating systems are filled with email addresses. They come from Windows binaries, SSL certificates and sample documents. Most of these email addresses, particularly those that occur the most frequently, such as someone@example.com, are not relevant to the case. It is important to be able to suppress those email addresses not relevant to the case. To address this problem, \bulk provides two approaches. \\

First, \bulk allows users to build a stop list or use an existing one available for download. These stop lists are used to recognize and dismiss the email addresses that are native to the Operating System. This approach works well for email addresses that are clearly invalid, such as someone@example.com. For most email addresses, however, you will want to stop them in some circumstances but not others. For example, there are over 20,000 Linux developers, you want to stop their email addresses in program binaries, not in email messages.\\

To address this problem, \bulk uses context-sensitive stop lists. Instead of a stop list of features, this approach uses the feature+context. The following example is an excerpt from a context-sensitive stop list file.

\lstset{style=customfile}
\begin{lstlisting}
ubuntu-users@lists.ubuntu.com   Maint\x0A935261357\x09ubuntu-users@lists.ubuntu.com\x0
ubuntu-motu@lists.ubuntu.com    untu_\x0A923867047\x09ubuntu-motu@lists.ubuntu.com\x09
pschiffe@redhat.com     Peter Schiffer <pschiffe@redhat.com> - 0.8-1.1N\x94/\xC0-
phpdevel@echospace.com  :  Vlad Krupin <phpdevel@echospace.com>\x0AMAINTENANCE:
anholt@freebsd.org      34-GZIP-1021192\x09anholt@freebsd.org\x09r: EricAnholt
ubuntu-motu@lists.ubuntu.com     http\x0A938966489\x09ubuntu-motu@lists.ubuntu.com\x09
\end{lstlisting}

The context for the feature is the 8 characters on either side of the feature. Each ``stop list'' entry is the feature+context. This ignores Linux developer email addresses in Linux binaries. The email address will be ignored if found in that context but reported if it appears in a different context. \\

There is a context-sensitive stop list for Microsoft Windows XP, 2000, 2003, Vista and several Linux systems. The total stop list is 70 MB and includes 628,792 features in a 9 MB zip file. The context-sensitive stop list prunes many of the OS-supplied features. By applying it to the domexusers HD image (the image can be downloaded at \url{http://http://digitalcorpora.org/corp/nps/drives/nps-2009-domexusers/}, the number of emails found went from 9,143 down to 4,459. This significantly reduces the amount of work to be done by the investigator. Figure ~\ref{fig:withandwithoutstoplist} shows how the histogram of email addresses differs when \bulk is run with and without the context-sensitive stop list. The context-sensitive stop list built for the various operating systems described above can be downloaded from \url{http://digitalcorpora.org/downloads/bulk_extractor}. The file will have the words ``stoplist'' in it somewhere. The current version as of publication of this manual is called \texttt{bulk\_extractor-3-stoplist.zip}.

\begin{figure}
	\includegraphics[scale=0.65]{otherPics/WithWithoutStopList.jpg}
	\caption{Email Histogram Results With and Without the Context-Sensitive Stop List. Results from the Domexusers HD image.}
	\label{fig:withandwithoutstoplist}
\end{figure}

It should be noted that \bulk does allow the users to create stop lists that are not context sensitive. A stop list can simply be a list of words that the user wants \bulk to ignore. For example, the following three lines would constitute a valid stop list file: 
\begin{verbatim}
abc@google.com
ignore@microsoft.com
www.google.com
\end{verbatim}

However for the reasons stated above, it is recommended that users rely on context-sensitive stop lists when available to reduce the time required to analyze the results of a \bulk run.\\

Stopped results are not completely hidden from users. If stopped feature are discovered, they will be written to the appropriate category feature file with the extension \texttt{\_stopped.txt}. For example, stopped domain names that are found in the disk image will be written to \texttt{domain\_stopped.txt} in the output directory. The stopped files serve the purpose of allowing users to verify that \bulk is functioning properly and that the lists they have written are being processed correctly.

\subsection{Using an Alert List}
Specific words or features in a given context might be important to a user's investigation. The alert list can contain a list of words and/or feature filenames, and when a match is found, it will alert the user. The way the feature file alert works is similar to how they are used for context-sensitive stop lists. It will only alert on a specified feature when it's found in the specified context.\\

A sample alert list file might look like the following:
\begin{verbatim}
abc@google.com
SilentFury2012
www.maliciousintent.com
\end{verbatim}

While this list does not appear to help in any particular investigation, it demonstrates that you can specify distinct words that are important to their analysis. Results containing the alert list information are found in the file \texttt{alert.txt} in the \bulk output directory.

\subsection{The Importance of Compressed Data Processing}
\label{compressedProcessing}

Many forensic tools frequently miss case-critical data because they do not examine certain classes of compressed data. For example, a recent study of 1400 drives found thousands of email addresses that were compressed (and happened to be in unallocated space).\cite{encodedEvidence}. Without looking at all the data on each drive and optimistically decompressing it, critical features might be missed. Compressed email addresses, such as those in a GZIP file, do not look like email addresses to a scanner; they must first be decompressed to be identified. Although some of these features are from software distributions, many are not. Table ~\ref{tab:encodings2} shows the kinds of encodings that can be decoded by \bulk \cite{encodedEvidence}.

\begin{table}[!ht]
\centering
\caption{The kinds of encodings that can be decoded by \bulk and the amount of context required for the decoding}
\label{tab:encodings2}
\begin{tabular}{|p{2 cm}|p{9 cm}|}
\hline \hline
\textbf{Encoding} & \textbf{Can be decoded when \bulk finds} \\
\hline
GZIP & The beginning of a zlib-compressed stream\\
\hline
BASE64 & The beginning of a BASE64-encoded stream \\
\hline
HIBER & Any fragment of a hibernation file can generally be decompressed, as each Windows 4k page is separately compressed and the beginning of each compressed page in the hibernation file is indicated by a well-known sequence\\
\hline
PDF & Any PDF stream compressed with ZLIB bracketed by \textit{stream} and \textit{endstream} \\
\hline
ZIP & The local file header of a ZIP-file component \\
\hline
\end{tabular}
\end{table}


The reason that users must be aware of this is because users have a tendency to want to enable and disable scanners for specific uses, but one can unintentionally compromise the results. For example, if a user only wants to find email addresses, they may try to turn off all scanners except the email scanner. This will find some email addresses. However, it will miss the email addresses on the media that are only present in compressed data. This is because scanners such as \textbf{\textit{zip}},  \textbf{\textit{rar}} and  \textbf{\textit{gzip}} will not be running. Those scanners each work on a different type of compressed data. For example, the  \textbf{\textit{gzip}} scanner will find GZIP compressed data, decompress it and then pass it other scanners to search for features. In that way, GZIP compressed emails can be processed by \bulk. \\

The  \textbf{\textit{pdf}} scanner is another type of scanner that finds text that otherwise wouldn't be found. While PDF files are human readable, they are not readable but many software tools and scanners because of their formatting. The  \textbf{\textit{pdf}} scanner extracts some kinds of text found within PDFs and then passes that text on to other scanners for further processing. Many typical disk images include PDF files, so most users will want to have this scanner enabled (as it is by default).\\

Finally, the  \textbf{\textit{hiber}} scanner decompresses Windows hibernation files. If the disk image being analyzed is from a Windows system, \bulk users will want that turned on (as it is by default). The scanner is very fast, however, so it will not significantly decrease performance on non-Windows drives.

\section{Use Cases for \bulk}
There are many digital forensic use cases for \bulk --- more than we can enumerate within this manual. In this section we highlight some of the most common uses of the system. Each case discusses which output files, including feature files and histograms, are most relevant to these types of investigations.  In \textbf{\Autoref{Examples}}, \textbf{\nameref{Examples}}, we provide more detailed walk-throughs and refer back to these use cases with more detailed output file information.

\subsection{Malware Investigations}
\label{malware}
Malware is a programmatic intrusion. When performing a malware investigation, users will want to look at executables, information that has been downloaded from web-based applications and windows directory entries (for Windows-specific investigations). \bulk enables this in several ways. \\

First, \bulk finds evidence of virtually all executables on the hard drive including those by themselves, those contained in ZIP files, and those that are compressed. It does not give you the hash value of the full file, rather, it gives the hash of just the first 4KB of the file. Our research has shown that the first 4KB is predictive because most executables have a distinct hash value for the first 4KB of the file \cite{hashEncoding}. Additionally, many of these files may be fragmented and looking at the first 4KB will still provide information relevant to an investigation because fragmentation is unlikely to happen before the first 4KB. The full hash of a fragmented file is not available in \bulk.\\ 

Several output feature files produced by \bulk contain relevant and important information about executables. These files include:
\begin{itemize}
\item \texttt{elf.txt} --- This file (produced by the \textbf{\textit{elf}} scanner) contains information about ELF executables that can be used to target Linux and Mac OS X systems.
\item \texttt{winprefetch.txt} --- This file (produced by the \textbf{\textit{winprefetch} scanner} lists the current and deleted files found in the Windows prefetch directory.
\end{itemize}
The XML in these feature files is too complicated to review without using other applications. The recommended way to analyze the executable output is to use a third party tool that analyzes executables or pull the results into a spreadsheet. In a spreadsheet, one column could contain the hash values and those values can be compared against a database of executable hashes. There is also a python tool that comes with \bulk called \textbf{identify\_filenames.py} that can be used to get the full filename of the file. The python tool is discussed in more detail in \textbf{\Autoref{PostProcessing}}.\\

For Windows specific malware investigations, the files \texttt{winpe.txt} and \texttt{winprefetch.txt} are very useful. They are produced by the \textbf{\textit{winpe}} and \textbf{\textit{winprefetch}} scanners respectively.  Windows Prefetch shows files that have been prefetched in the Windows prefetch directory and shows the deleted files that were found in unallocated space. The Windows PE feature file shows entries related to Windows portable executable files. \\

JSON, the JavaScript Object Notation, is a lightweight data-interchange format. Websites tend to download a lot of information using JSON. The output file \texttt{json.txt}, produced by the \textbf{\textit{json}} scanner, can be useful for malware investigations and analysis of web-based applications. If a website has downloaded information in JSON format, the JSON scanner may find that information in the browser cache. 

\subsection{Cyber Investigations}
\label{cyber}
Cyber investigations may scan a wide variety of information types. A few unifying features of these investigations are the need to find encryption keys, hash values and information about ethernet packets. \bulk provides several scanners that produce feature files containing this information.\\

For encryption information, the following feature files may be useful:
\begin{itemize}
\item \texttt{aes.txt} --- AES is an encryption system. Many implementations leave keys in memory that can be found using an algorithm invented at Princeton University. \bulk provides an improved version of that algorithm to find AES keys in the  \textbf{\textit{aes}} scanner. When it scans memory, such as swap files or decompressed hibernation files, it will identify the AES keys. The keys can be used for software that will decrypt AES encrypted material. 
\item \texttt{hex.txt} --- The  \textbf{\textit{base16}} scanner decodes information that is stored in Base16, breaking it into the corresponding hexidecimal values. This is useful if you are looking for AES keys or SHA1 hashes. This scanner only writes blocks that are of size 128 and 256 because they are the sizes used for encryption keys. The feature file is helpful if the investigator is looking for people who have emailed encryption keys or hash values in a cyber investigation.
\end{itemize}

Additionally, the  \textbf{\textit{base64}} scanner is important for cyber investigations because it looks mostly at email attachments that are coded in Base64. The information found in these attachments will be analyzed by other scanners looking for specific features.\\

The \textbf{\textit{windirs}} scanner finds Windows FAT32 and NTFS directory entries and will also be useful for cyber investigations involving Windows machines, as they may be indicators of times that activity took place.\\

Finally, the files \texttt{ether.txt}, \texttt{ip.txt}, \texttt{tcp.txt} and \texttt{domain.txt} are all produced by the  \textbf{\textit{net}} scanner. It searches for ethernet packets and memory structures associated with network data structures in memory. It is important to note that tcp connections have a lot of false positives and many of the information found by this scanner will be false. Investigators should be careful with the interpretation of these feature files for that reason.

\subsection{Identity Investigations}
\label{identity}
Identity investigations may be looking for a wide variety of information including email addresses, credit card information, telephone numbers, geographical information and keywords. For example, if the investigator is trying to find out of who a person is and who their associates are, they will be looking at phone numbers, search terms to see what they are doing and emails to see who they are communicating with. \\

The  \textbf{\textit{accts}} scanner is very useful for identity investigations. It produces several feature files with identity information including:
\begin{itemize}
\item \texttt{ccn.txt} --- credit card numbers
\item \texttt{ccn\_track2.txt} - credit card track two information -  relevant information if someone is trying to make physical fake credit cards
\item \texttt{pii.txt} - personally identifiable information including birth dates and social numbers
\item \texttt{telephone.txt} - telephone numbers
\end{itemize}

The  \textbf{\textit{kml}} and  \textbf{\textit{gps}} scanner both produce GPS information that give information about a person in a certain area or link to what they have been doing in a certain area. Both of these scanners write to \texttt{gps.txt}. KML is a format used by Google Earth and Google Map files. This scanner searches in that formatted data for GPS coordinates. The  \textbf{\textit{gps}} scanner looks at Garmin Trackpoint formatted information and finds GPS coordinates  in that data.\\

The  \textbf{\textit{email}} scanner looks for email addresses in all data and writes that to \texttt{email.txt}. The  \textbf{\textit{vcard}} scanner looks at vCard data, an electronic business card format, and finds names, email addresses and phone numbers to write to the respective feature file.\\

The are multiple url files including \texttt{url.txt}, \texttt{url\_facebook-address.txt}, \texttt{url\_facebook-id.txt}, \texttt{url\_microsoft-live.txt}, \texttt{url\_searches.txt} and \texttt{url\_services.txt} that are produced by the \textbf{\textit{email}} scanner. They are useful for looking at what websites a person has visited as well as the people they are associating with.\\

An important aspect of identity investigations (as well as other types) is the ability to search the data for a list of keywords. \bulk provides the capability to do that through two different means. First, the  \textbf{\textit{find}} scanner is a simple regular expression finder that uses regular expressions. The \textbf{\textit{find}} scanner looks through the data for anything listed in the global find list. The format of the find list should be rows of regular expressions while any line beginning with a \# is considered a comment. The following is an excerpt from a sample find list file:
\lstset{style=customfile}
\begin{lstlisting}
# This is a comment line
\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b
# another comment line
/^[a-z0-9_-]{3,16}$/
\end{lstlisting}
%$
The first regular expression from the above example, beginning with \texttt{\textbackslash b}, looks for the following in order: a word boundary followed a digit repeated between 1-3 times, a digit repeated between 1-3 times, a digit repeated 1-3 times, a '.', a digit repeated 1-3 times, a digit repeated 1-3 times and the end of the word boundary. That regular expression would find, for example, the sequence 2219.889 separated out from other text by a word boundary.\\

The second regular expression from the above example, beginning with \texttt{/} looks for the following in order: a '/', the beginning of a line, repeats of any character in lowercase a-z, 0-9, '\_', or '-', repeated 3 to 16 times, and the end of the line followed by '\textbackslash.' That expression would find, for example, the following sequence:
\\
\textbackslash \\
284284284284 \\
/ \\
Regular expressions can be used to represent character and number sequences (or ranges of values) that might be of particular importance to an investigation. \\

The find list is sent in as input to \bulk using the ``-F \texttt{findlist}'' option. To run \bulk with a find list, the following basic parameters are required (where \texttt{findlist.txt} is the name of the find list):
\begin{Verbatim}[commandchars=\\\{\}]
\verbbf{bulk_extractor -F findlist.txt -o output mydisk.raw}
\end{Verbatim}

Another scanner, the \textbf{\textit{lightgrep}} scanner provides the same functionality as the \textbf{\textit{find}} scanner but it is much faster and provides more functionality. It is also a regular expression scanner that looks through the buffers and matches in the global find list. A syntax sheet of regular expressions that might be helpful to users in creating a find list to be used by the Lightgrep Scanner is shown in Figure ~\ref{fig:lightgrepSyntax}. \\

\begin{figure}
	\includegraphics[scale=.80]{otherPics/LightgrepCheatSheet.pdf}
	\caption{Guide to Syntax Used by Lightgrep Scanner}
	\label{fig:lightgrepSyntax}
\end{figure}

The \textbf{\textit{lightgrep}} scanner uses the Lightgrep library from Lightbox Technologies. An open source version of that library can be downloaded from \url{https://github.com/LightboxTech/liblightgrep}. Installation instructions are also available at the download site. The \textbf{\textit{lightgrep}} scanner is preferable because it looks for all regular expressions at once, on the first pass through the data. The \textbf{\textit{find}} scanner actually looks for each expression in the find list one at a time. For example, if the find list is a list of medical terms and diagnoses and \bulk is searching medical records, the \textbf{\textit{find}} scanner looks for each term in each piece of data on one pass through, one at a time. A list of 35 expressions would require 35 passes through the data. The \textbf{\textit{lightgrep}} scanner will search a given buffer for all of the medical terms at once, in one pass through. \\

If the Lightgrep library is installed and the find list is provided to \bulk, it will run the \textbf{\textit{lightgrep}} scanner. If not, it will use the \textbf{\textit{find}} scanner. Neither scanner needs to be enabled by the user specifically, calling \bulk with the find list will automatically enable the appropriate scanner. However, we do not recommend using the find list without the Lightgrep library --- it will make \bulk run very slowly because each find search will be sequentially executed. This will provide an exponential slow-down.\\

Investigators looking for identity information may rely heavily on the find list to search for specific names, numbers or keywords relevant to the investigation. The features found by the \textbf{\textit{find}} or \textbf{\textit{lightgrep}} scanner will be written to the files \texttt{find.txt} and \texttt{lightgrep.txt} respectively.

\subsection{Password Cracking}
\label{PasswordCracking}
If an investigator is looking to crack a password, the \textbf{\textit{wordlist}} scanner can be useful. It generates a list of all the words found on the disk that are between 6 and 14 characters. Users can change the minimum and maximum size of words by specifying options at run-time but we have found this size range to be optimal for most applications. Because the \textbf{\textit{wordlist}} scanner is disabled by default, users must specifically enable it at run-time when needed. To do that, run the following command:
\begin{Verbatim}[commandchars=\\\{\}]
\verbbf{bulk_extractor -e wordlist -o output mydisk.raw}
\end{Verbatim}
This will produce two files useful for password cracking, \texttt{wordlist\_histogram.txt} and \texttt{wordlist.txt}.  These files will contain large words that can be used to recommend passwords.

\subsection{Analyzing Imagery Information}
\label{imagery}
In an investigator needs to specifically analyze imagery, for something such as a child pornography investigation, the \textbf{\textit{exif}} scanner would be useful. It finds JPEGs on the disk image and then carves the encoded ones that might be in, for example, ZIP files or hibernation files. It writes the output of this carving to \texttt{jpeg\_carved.txt}.

\subsection{Using \bulk in a Highly Specialized Environment}
If using \bulk in a specialized environment, two specific features might be useful.  The first is the option to include a banner on each output file created by \bulk. The banner file, specified in the example command below as \texttt{banner.txt} could include a security classification of the output data. When \bulk is run with the command specified below, the data in the banner file will be printed at the top of each output file produced.
\begin{Verbatim}[commandchars=\\\{\}]
\verbbf{bulk_extractor -b banner.txt -o output mydisk.raw}
\end{Verbatim}

The second feature might be useful to users in a specialized environment is the ability to develop plug-ins. Plug-ins in \bulk are external scanners that an individual or organization can run in addition to the open source capabilities provided with the \bulk system. The plug-in system gives the full power of \bulk to external developers, as all of \bulk's native scanners are written with the plug-in system. This power gives third party developers the ability to utilize proprietary or security protected algorithms and information in \bulk scanners. It is worth noting that all scanners installed with \bulk use the plug-in system, \bulk is really just a framework for running plug-ins.  The separate publication \textbf{Programmers Manual for Developing Scanner Plug-ins} \cite{programmersmanual} provides specific details on how to develop and use plug-ins with \bulk.

\section{Tuning \bulk}
All data that \bulk processes is divided into buffers called sbufs. Buffers created from disk images are created with a pre-determined size (bufsize). The buffer includes a page and an overlap area. As shown in Figure ~\ref{fig:margindepiction}, the pages overlap with each other in the red region. The red overlap region is called the margin. \bulk scans the pages one-by-one looking for features. Pages overlap with each other so that \bulk won't miss any features that cross from one page into another across boundaries.\\

Users may be looking for potentially large features that are bigger than the buffer size or that overlap into the margin. In that case, they may want to adjust the margin size or buffer size.  For example, if the input data includes a 30 MB ZIP file (possibly a software program), \bulk won't find features in the program because it overlaps the margins. To find features of that size, the margin size must be increased. \\

\begin{figure}
	\includegraphics[scale=0.60]{archPics/margindepiction.pdf}
	\caption{Image Processor divides the disk image into buffers. Each buffer is the size of a page (\textit{pagesize}) with a buffer overlap in an area called the margin. (\textit{marginsize} is equal to \textit{bufsize-pagesize}). The buffers overlap with each other to ensure all information is processed.}
	\label{fig:margindepiction}
\end{figure}

To adjust the page size, the following usage options need to be included where NN should be set to the size (default page size is 16777216):
\begin{Verbatim}[commandchars=\\\{\}]
\verbbf{bulk_extractor -G NN -o output mydisk.raw}
\end{Verbatim}

To adjust the margin size, the following usage options need to be included where NN should be set to the size (default margin size is 4194304):
\begin{Verbatim}[commandchars=\\\{\}]
\verbbf{bulk_extractor -g NN -o output mydisk.raw}
\end{Verbatim}

\bulk provides many other tuning capabilities that are primarily recommended for users doing advanced research. Many of those options relate to specifying file sizes for input or output, specifying block sizes, dumping the contents of a buffer or ignoring certain entries. Those options are all found in the output of the -h input to \bulk and listed in \textbf{\Autoref{HelpOutput}}.

\section{Post Processing Capabilities}
\label{PostProcessing}
There are two Python programs useful for post-processing the \bulk output. Those programs are \textbf{bulk\_diff.py} and \textbf{identify\_filenames.py}. To run either of these programs, you must have Python version 2.7 or higher installed on your system. On Linux and Mac systems, the \bulk python programs are located in the directory \textit{./python} under the main \bulk installation. \\

\subsection{bulk\_diff.py: Difference Between Runs}
The program \textbf{bulk\_diff.py} takes the results of two \bulk runs and shows the differences between the two runs. This program essentially tells the difference between two disk images. It will note the different features that are found by \bulk between one image and the next. It can be used, for example, to easily tell whether or not a computer user has been visiting websites they are not supposed to by comparing a disk image from their computer from one week to the next. To run the program, users should type the following, where \textit{pre} and \textit{post} are both locations of two \bulk output directories:
\begin{Verbatim}[commandchars=\\\{\}]
\verbbf{bulk\_diff.py <pre> <post> }
\end{Verbatim} 
Note, Linux and Mac OS X users may have to type \textbf{python2.7}, \textbf{python3}, or \textbf{python3.3} before the command, indicating the version of Python installed on your machine. An example use of the \textbf{bulk\_diff.py} program can be found in \Autoref{Examples}.


\subsection{identify\_filenames.py: Identify File Origin of Features}
The program \textbf{identify\_filenames.py} operates on the results of \bulk run and identifies the filenames (where possible) of the features that were found on the disk image. The user can run this program on one or all of the features file produced by a given run. It can be used, for example, to find the full content of an email when references to its contents are found in one of the feature files. Often email features are relevant to an investigation and an investigator would like to be able to view the full email.\\

To run this program, users will need the program \textbf{fiwalk} installed on their machine or have a DFXML file generated by \textbf{fiwalk} that corresponds to the disk image. \textbf{fiwalk} is part of the \textbf{SleuthKit} and can be installed by installing \textbf{Sleuthkit}, available at \url{http://www.sleuthkit.org/}.\\

The \textbf{identify\_filenames.py} program provides various usage options but to run the program on all feature files produced by a \bulk run, the user should type the following (where ``bulkoutputdirectory'' is the directory containing the output of a \bulk run and ``idoutput'' will contain the annotated feature files after the program runs):
\begin{Verbatim}[commandchars=\\\{\}]
\verbbf{identify\_filenames.py --all bulkoutputdirectory idoutput}
\end{Verbatim} 
Note, Linux and Mac OS X users may have to type \textbf{python2.7}, \textbf{python3}, or \textbf{python3.3} before the command, indicating the version of Python installed on your machine.  An example use of the \textbf{bulk\_diff.py} program can be found in \Autoref{Examples}.

\section{Worked Examples}
\label{Examples}
\input{./BEWorkedExamples.tex}

\section{Troubleshooting}
\label{DebuggingBulk}
Every forensic tool crashes at times because the tools are routinely used with data fragments, non-standard codings, etc. One major issue is that the evidence that makes the tool crash typically cannot be shared with the developer. The \bulk system implements checkpointing to protect the user and the results. \bulk checkpoints the current page in the file \texttt{report.xml}. After a crash, the user can just hit the up-arrow at the command line prompt and return. \bulk will restart at the next page. \\

All \bulk users should join the \bulk users Google group for more information and help with any issues encountered. To join, send an email to \textbf{bulk\_extractor-users+subscribe@googlegroups.com}.  \\

For the most part, the only kind of debugging \bulk users should be doing is turning off scanners. If \bulk crashes repeatedly on a data set, the scanners can all be disabled and then turned back on, one by one, until it crashes again. Then, the user can report the specific scanner that made \bulk crash on their disk image. In general, users who experience crashes should feel free to report issues and problems to the developers via the Google users group.\\

Users running the 32-bit version of \bulk may occasionally encounter memory allocation errors. This problem is more likely to occur on machines with a greater number of cores. Our testing has shown this to be an issue using one of our test data sets on a 32-bit machine with 12 cores.  If the user encounters memory allocation errors with \bulk they will likely see an error similar to the following:

\begingroup
\footnotesize
\begin{Verbatim}
bulk_extractor scan error: 'std::exception Scanner: gzip Exception:
std::bad_alloc sbuf.pos0: (|21894266880) bufsize=20971520'
\end{Verbatim}
\endgroup 
Memory allocation errors such as the one shown above will contain the phrase ``bad\_alloc'' somewhere in the message. If the user encounters this error, they should try running \bulk with fewer threads. For example, the following command will run \bulk with only 4 threads (the -j option changes this parameter):
\begin{Verbatim}[commandchars=\\\{\}]
\verbbf{bulk_extractor -j 4 -o output mydisk.raw}
\end{Verbatim}
Reducing the number of threads and re-running the program should eliminate the problem.\\

Users may encounter errors if they are processing a large disk image and trying to write the output of \bulk to an output file directory on a smaller drive. In that case the user might see an error similiar to the following:

\begingroup
\footnotesize
\begin{Verbatim}
bulk_extractor version: 1.5.0
Input file: G:\nps-2011-2tb\nps-2011-2tb.E01
Output directory: C:\Users\Mark Richer\Documents\BE Testing\OFD nps-2011-2tb 64bit
Disk Size: 2000054960128
Threads: 12
DISK FULL
DISK FULL
DISK FULL
*** carve: Cannot write(pos=7,0 len=24724184): No space left on device
DISK FULL
DISK FULL
DISK FULL
DISK FULL
DISK FULL
*** carve: Cannot write(pos=7,0 len=24724198): No space left on device
*** carve: Cannot write(pos=7,0 len=49160): No space left on device

*** carve: Cannot create C:\Users\Mark Richer\Documents\BE Testing\OFD nps-2011-2tb
64bit/kml/000/426602508288-ZIP-0.kml: No space left on device

Could not make directory C:\Users\Mark Richer\Documents\BE Testing\OFD nps-2011-2tb
64bit/kml/001: No space left on device

Phase 3. Creating Histograms
Cannot open histogram output file: C:\Users\Mark Richer\Documents\BE Testing\OFD
nps-2011-2tb 64bit/ccn_track2_histogram.txt

Elapsed time: 45111.4 sec.
Overall performance: 44.3359 MBytes/sec
Total email features found: 6716934
\end{Verbatim}
\endgroup
If this situation is encountered, the solution is to run \bulk with an output directory on a machine with more available disk space so that \bulk has room to create all the output files and directories required. 

\section{Related Reading}
There are numerous articles and presentations available related to digital forensics, specifically \bulk, and its practical and research applications. Some of those articles are specifically cited throughout this manual. Other useful references include but are not limited to:
\begin{itemize}
\item Garfinkel, S. File Cabinet Forensics, Journal of Digital Forensics, Security and Law, Vol 6(4). \url{http://www.jdfsl.org/subscriptions/abstracts/JDFSL-V6N4-column-Garfinkel.pdf}
\item Garfinkel, S. Every Last Byte. J. of Digital Forensics, Security and Law, 6:7–8. Column. \url{http://www.jdfsl.org/subscriptions/abstracts/column-v6n2-Garfinkel.htm}
\item Phillips, Kenneth N; Aaron Pickett; Simson Garfinkel, Embedded with Facebook: DoD Faces Risks from Social Media, CrossTalk, May/June 2011. \url{http://www.dtic.mil/cgi-bin/GetTRDoc?AD=ADA542587}
\item Rowe, Neil, Schwamm, Riqui, Garfinkel, Simson. Language Translation for File Paths, DFRWS 2013, Aug 4-7, 2013. Monterey, CA. \url{http://www.dfrws.org/2013/proce}\newline\url{edings/DFRWS2013-5.pdf}
\item Garfinkel, S., Nelson, A., Young, J., ``A General Strategy for Differential Forensic Analysis'', DFRWS 2012, Aug. 6-8, 2012, Washington, DC. \url{http://www.dfrws.org/2012/proceedings/DFRWS2012-6.pdf}
\item Garfinkel, S., ``Lessons Learned Writing Computer Forensics Tools and Managing a Large Digital Evidence Corpus'', DFRWS 2012, Aug. 6-8, 2012, Washington, DC. \url{http://simson.net/clips/academic/2012.DFRWS.DIIN382.pdf}
\item N. C. Rowe and S. L. Garfinkel, Finding anomalous and suspicious files from directory metadata on a large corpus. 3rd International ICST Conference on Digital Forensics and Cyber Crime, Dublin, Ireland, October 2011. In P. Gladyshev and M. K. Rogers (eds.), Lecture Notes in Computer Science LNICST 88, Springer-Verlag, 2012, pp. 115-130. \url{http://simson.net/clips/academic/2012.IICDFCC.Anomalous.pdf}
\item Presentation - Using \bulk for digital forensics triage and cross-drive analysis, DFRWS 2012. \url{http://digitalcorpora.org/downloads/bulk_extractor/doc/2012-08-08-bulk_extractor-tutorial.pdf}
\item Presentation - Digital Signatures: Current Barriers, Invited Talk, 10th Symposium on Identity and Trust on the Internet, Gaithersburg, MD, 2011. \url{http://middleware.internet2.edu/idtrust/2011/slides/07-digital-signatures-current}\newline\url{-barriers-garfinkel.pdf}
\item Courrejou, Timothy and Simson Garfinkel. A comparative analysis of file carving software. Technical Report NPS-CS-11-006, Naval Postgraduate School, September 2011. \url{http://www.dtic.mil/cgi-bin/GetTRDoc?Location=U2&doc=GetTRDoc.pdf&AD=}\newline\url{ ADA550119}
\end{itemize}


\bibliographystyle{acm} 
\bibliography{references}

\newpage
\appendix
\appendixpage



\section{Output of \bulk Help Command}
\label{HelpOutput}

\begingroup
\footnotesize
\texttt{C:\textbackslash \textgreater \textbf{bulk\_extractor -h}}
\endgroup

\begingroup
\footnotesize
{
\fontfamily{courier}\selectfont
\verbatiminput{BEUsage.txt}
}
\endgroup

\end{document}
