# http://codex.wordpress.org/Editing_wp-config.php#Debug
define('WP_DEBUG', true);
# http://codex.wordpress.org/Editing_wp-config.php#Configure_Error_Log
@ini_set('log_errors','On');
@ini_set('display_errors','Off');
@ini_set('error_log',dirname(__FILE__).'/logs/php_error.log');

I wrote a 50-line Ruby script to list out the duplicate files. The script uses SQLite3 and SHA-1 digests to identify multiple copies of each file in a directory. The end result was 2K duplicate files and 20 GB of freed disk space. Woot.

Here’s the script:

#!/usr/bin/ruby

# deduplicate_files.rb

require 'rubygems'
require 'sqlite3' # Look at 'http://github.com/luislavena/sqlite3-ruby' then do 'sudo gem install sqlite3-ruby'
require 'digest/sha1'
require 'pathname'

# Pass in the directory or assume the current one.
arg = ARGV[0] || "."
root_path = Pathname.new(arg).realpath.to_s
puts "Examining #{root_path}"

# Create a SQLite3 database in the current directory.
db = SQLite3::Database.new("deduplicate_files.db")
db.execute("create table files(digest varchar(40),path varchar(1024))")

# Recursively generate hash digests of all files.
Dir.chdir("#{root_path}")
current_file = 0
Dir['**/*.*'].each do |file|
  path = "#{root_path}/#{file}"
  # Ignore non-existent files (symbolic links) and directories.
  next if !File.exists?("#{path}") || File.directory?("#{path}")
  # Create a hash digest for the current file.
  digest = Digest::SHA1.new
  File.open(file, 'r') do |handle|
    while buffer = handle.read(1024)
      digest << buffer
    end
  end
  # Store the hash digest and full path in the database.
  db.execute("insert into files values(\"#{digest}\",\"#{path}\")")
  # Print out every Nth file.
  puts "[#{digest}] #{path} (#{current_file})" if current_file % 100 == 0
  current_file = current_file + 1
end

# Loop through digests.
db.execute("select digest,count(1) as count from files group by digest order by count desc").each do |row|
  if row[1] > 1 # Skip unique files.
    puts "Duplicates found:"
    digest = row[0]
    # List the duplicate files.
    db.execute("select digest,path from files where digest='#{digest}'").each do |dup_row|
      puts "[#{digest}] #{dup_row[1]}"
    end
  end
end

NSDictionary *infoPlistDictionary = [[NSBundle mainBundle] infoDictionary];
BOOL doesKeyExistInPlist = [infoPlistDictionary valueForKey:PLIST_KEY] != nil;
BOOL isKeyTrueInPlist = [[infoPlistDictionary valueForKey:PLIST_KEY] boolValue] != NO;
NSUserDefaults *defaults = [NSUserDefaults standardUserDefaults];
BOOL doesKeyExistInPrefs = [defaults valueForKey:PLIST_KEY] != nil;
BOOL isKeyTrueInPrefs = [[defaults valueForKey:PLIST_KEY] boolValue] != NO;

Font: When I clicked the link, the URL’s font changed. To keep it consistent, I needed to set it myself. I added the following code to Apple’s NSAttributedString category from the link above:

[attrString addAttribute:NSFontAttributeName value:[NSFont fontWithName:@"Lucida Grande" size:13] range:range];

Cursor: With Apple’s code, the pointing hand cursor only appeared after I clicked the link. To get the correct cursor before the link is clicked, I followed the advice from a cocoa-dev post and subclassed NSTextField to override resetCursorRects.

When consumers have a great user experience, they don’t notice. They don’t wonder how you designed the product so well. They just use it and move on, but that’s a good thing. They shouldn’t feel bad while using your product. Don’t make users feel bad because eventually they’ll choose a better user experience as a coping mechanism.

User experience: Because your users internalize your product’s failures.

If the tool nm and its wealth of information seems impressive, class-dump is amazing. Feed it an executable and get back a complete list of class names, internal variables, and methods, even as a set of header files, thanks to its deep understanding of Objective-C and Mach-O. Apple’s product might be stripped of their symbols, but class-dump can list their internal structure.

I created a template application, added a method (shouldShowInClassDump), and built it with all the symbols stripped out. Running class-dump on the resulting Release executable revealed the following:

/*
 *     Generated by class-dump 3.3.1 (64 bit).
 *
 *     class-dump is Copyright (C) 1997-1998, 2000-2001, 2004-2009 by Steve Nygard.
 */

#pragma mark -

/*
 * File: build/Release/SampleApp.app/Contents/MacOS/SampleApp
 * Arch: Intel x86-64 (x86_64)
 *
 *       Objective-C Garbage Collection: Unsupported
 */

@interface SampleAppAppDelegate : NSObject
{
    NSWindow *window;
}

- (void)applicationDidFinishLaunching:(id)arg1;
- (BOOL)shouldShowInClassDump:(id)arg1;
@property NSWindow *window; // @synthesize window;

@end

To see how applications compared in aggregate, I wrote a simple ruby script to walk through /Applications and sum up the number of lines of headers from class-dump and the number of @interface declarations. Both are crude measures of complexity.

Top 10 by Header Lines

37968	iPhoto
37053	Aperture
21228	iWeb
19307	Keynote
16564	iMovie
16235	Coda
15625	Keynote
15111	Grapher
14735	Pages
14724	iCal

Top 10 by Interfaces

1203	Aperture
1066	iWeb
1004	Grapher
 967	iPhoto
 859	Path Finder
 748	Keynote
 730	iMovie
 664	LaunchBar
 649	iCal
 646	Coda

Top 10 by Lines Per Interface

39.54	Retrospect
39.33	Grab
39.26	iPhoto
37.72	MemoryMiner
37.28	Papers
35.72	iTunes
35.65	Sequel Pro
34.71	Console
32.72	iChat
32.38	GarageBand

Apple’s tools are clearly complex, with iPhoto and Aperture leading by almost a factor of two in header lines, but apps like Panic’s Coda hold their own. And Retrospect topping the final list is obviously an indictment of my coding abilities.

Ruby Script

#!/usr/local/bin/ruby19

# analyze_apps.rb
# * Analyze dump from 'class-dump path_to_executable' to collect statistics: Interfaces, Lines

# Collect app paths.
CLASS_DUMP_PATH = "/usr/local/bin/class-dump"
apps = []
apps_folders = ["/Applications", "/Applications/iWork '08", "/Applications/iWork '09", "/Applications/Microsoft Office 2004", "/Applications/Microsoft Office 2008", "/Applications/Utilities"]
apps_folders.each do |app_folder|
  app_list = Dir.entries(app_folder)
  app_list.each do |app|
    apps << "#{app_folder}/#{app}"
  end
end

# Loop through apps.
apps.each do |app|
  # Check that the app exists.
  if Dir.exists?("#{app}") && app.include?(".app")
    app_array = app.split("/")
    app_name = app_array[app_array.count-1].split(".app")[0]
    exe_path = "#{app}/Contents/MacOS/#{app_name}"
    # Check that the executable exists.
    if File.exists?(exe_path)
      # Use 'classdump' to dump Mach-O header information.
      results = %x[#{CLASS_DUMP_PATH} "#{exe_path}"].split("\n")
      line_count = results.count
      interface_count = 0
      results.each do |line|
        if line.include?("@interface ")
          interface_count = interface_count + 1
          # puts "#{app_name}: #{line}"
        end
      end
      puts "#{app_name},#{app},#{interface_count},#{line_count}"
    end
  end
end

When Mac applications are compiled, Xcode has a setting to include or exclude debugging symbols: the “Strip Style”. When set to fully strip, the bundled executable (AppName.app/Contents/MacOS/AppName) contains very few symbols, but when all the symbols are included, the executable is a wealth of information. Unfortunately, Xcode’s default Release configuration includes all the symbols, leading to shipping applications with lots of symbol information, including method names and even source files. These are accessible with nm. Let’s see what we find for Retrospect Client (from my company):

Method Names

...
0000431d t -[PrefController awakeFromNib]
0000431d - 01 001f   FUN -[PrefController awakeFromNib]:f(0,1)
...

Source Files

...
0000276e - 01 0032    SO /Volumes/Leopard/projects/trunk/retrospect/client/mac/cpgui/AboutPanel.m
00004963 - 01 0032    SO /Volumes/Leopard/projects/trunk/retrospect/client/mac/cpgui/PasswordController.m
...

While interesting in isolation, the information is richer in aggregate. With a quick ruby script, we can rank the apps in /Applications by various metrics:

Top 10 by Methods

8857	Coda
8036	Evernote
7660	Cornerstone
6312	Pages
6054	Jing
5100	Papers
4805	Delicious Library 2
4589	Socialite
4563	Retrospect
4048	CSSEdit

Top 10 by Source Files

  304	Delicious Library 2
  295	LittleSnapper
  257	Socialite
  240	Evernote
  192	NetNewsWire
  158	Microsoft Document Connection
  144	Tweetie
  115	Retrospect
  105	Clipstart
   87	Transmit

Apple’s products are not in either list because they strip out the symbols for every major application. However, they apparently missed App Store’s Application Loader 1.1 (72), codenamed StarGazer:

...
00000000 - 00 0000    SO /Users/jfosback/jingle/iTMSTransporter/branches/iTMSTransporter-1.4/Applications/Stargazer/Source/ITunesPurpleSoftwareUploadDocumentController.m
...

Xcode Settings
Over half of the apps in my /Applications had no symbol information. The developers stripped it out, as described by Apple’s documentation on symbolizing crash dumps.

As I mentioned earlier, Apple’s default Release configuration does not strip out symbols. In fact, it leaves them all in, regardless of what “Strip Style” (STRIP_STYLE) is set to. The culprit is “Deployment Postprocessing” (DEPLOYMENT_POSTPROCESSING). When not enabled, Xcode ignores the setting for “Strip Style”. Apple describes this “Deployment Postprocessing” option in Xcode Build System Guide: Build Configuration and further in Xcode Project Management Guide: Building Products.

I wrote a quick sample application in Xcode and used the default Release configuration to generate the following dumps from nm:

No symbols are stripped (Xcode’s default Release configuration)

# Deployment Postprocessing: false
# Strip Style: any
[~/Desktop] $ nm -a SampleApp/build/Release/SampleApp.app/Contents/MacOS/SampleApp
0000000000000011 - 01 0000 ENSYM
0000000000000011 - 00 0000   FUN
0000000000000011 - 00 0000   FUN
00000001000018fd - 01 0000 BNSYM
0000000000000000 - 00 0000    SO
00000001000018dc - 01 0000 BNSYM
000000000000000a - 00 0000   FUN
000000000000000a - 01 0000 ENSYM
0000000000000000 - 01 0000    SO
0000000000000000 - 00 0000    SO
0000000000000000 - 01 0000    SO
0000000000000011 - 01 0000 ENSYM
00000001000018e6 - 01 0000 BNSYM
0000000000000006 - 00 0000   FUN
0000000000000006 - 01 0000 ENSYM
00000001000018ec - 01 0000 BNSYM
0000000100001e8b s  stub helpers
00000001000018e6 t -[SampleAppAppDelegate applicationDidFinishLaunching:]
00000001000018e6 - 01 0000   FUN -[SampleAppAppDelegate applicationDidFinishLaunching:]
00000001000018fd t -[SampleAppAppDelegate setWindow:]
00000001000018fd - 01 0000   FUN -[SampleAppAppDelegate setWindow:]
00000001000018ec - 01 0000   FUN -[SampleAppAppDelegate window]
00000001000018ec t -[SampleAppAppDelegate window]
0000000000000000 - 00 0000    SO /Users/bdunagan/Desktop/SampleApp/SampleAppAppDelegate.m
000000004be395c1 - 00 0001   OSO /Users/bdunagan/Desktop/SampleApp/build/SampleApp.build/Release/SampleApp.build/Objects-normal/x86_64/SampleAppAppDelegate.o
000000004be395c1 - 00 0001   OSO /Users/bdunagan/Desktop/SampleApp/build/SampleApp.build/Release/SampleApp.build/Objects-normal/x86_64/main.o
0000000000000000 - 00 0000    SO /Users/bdunagan/Desktop/SampleApp/main.m
                 U _NSApplicationMain
0000000100002660 D _NXArgc
0000000100002668 D _NXArgv
                 U _OBJC_CLASS_$_NSObject
0000000100002070 - 0a 0000 STSYM _OBJC_CLASS_$_SampleAppAppDelegate
0000000100002070 s _OBJC_CLASS_$_SampleAppAppDelegate
00000001000021c8 s _OBJC_IVAR_$_SampleAppAppDelegate.window
00000001000021c8 - 0b 0000 STSYM _OBJC_IVAR_$_SampleAppAppDelegate.window
                 U _OBJC_METACLASS_$_NSObject
0000000100002048 s _OBJC_METACLASS_$_SampleAppAppDelegate
0000000100002048 - 0a 0000 STSYM _OBJC_METACLASS_$_SampleAppAppDelegate
0000000100002678 D ___progname
0000000100000000 A __mh_execute_header
                 U __objc_empty_cache
                 U __objc_empty_vtable
0000000100002670 D _environ
                 U _exit
00000001000018dc t _main
00000001000018dc - 01 0000   FUN _main
                 U dyld_stub_binder
00000001000018a0 T start

Debugging symbols are stripped

# Deployment Postprocessing: true
# Strip Style: Debugging Symbols (STRIP_STYLE=debugging)
[~/Desktop] $ nm -a SampleApp/build/Release/SampleApp.app/Contents/MacOS/SampleApp
0000000100001e8b s  stub helpers
00000001000018e6 t -[SampleAppAppDelegate applicationDidFinishLaunching:]
00000001000018fd t -[SampleAppAppDelegate setWindow:]
00000001000018ec t -[SampleAppAppDelegate window]
                 U _NSApplicationMain
0000000100002660 D _NXArgc
0000000100002668 D _NXArgv
                 U _OBJC_CLASS_$_NSObject
0000000100002070 s _OBJC_CLASS_$_SampleAppAppDelegate
00000001000021c8 s _OBJC_IVAR_$_SampleAppAppDelegate.window
                 U _OBJC_METACLASS_$_NSObject
0000000100002048 s _OBJC_METACLASS_$_SampleAppAppDelegate
0000000100002678 D ___progname
0000000100000000 A __mh_execute_header
                 U __objc_empty_cache
                 U __objc_empty_vtable
0000000100002670 D _environ
                 U _exit
00000001000018dc t _main
                 U dyld_stub_binder
00000001000018a0 T start

Non-global symbols are stripped

# Deployment Postprocessing: true
# Strip Style: Non-Global Symbols (STRIP_STYLE=non-global)
[~/Desktop] $ nm -a SampleApp/build/Release/SampleApp.app/Contents/MacOS/SampleApp
                 U _NSApplicationMain
0000000100002660 D _NXArgc
0000000100002668 D _NXArgv
                 U _OBJC_CLASS_$_NSObject
                 U _OBJC_METACLASS_$_NSObject
0000000100002678 D ___progname
0000000100000000 A __mh_execute_header
                 U __objc_empty_cache
                 U __objc_empty_vtable
0000000100002670 D _environ
                 U _exit
                 U dyld_stub_binder
0000000005614542 - 00 0000   OPT radr://5614542
00000001000018a0 T start

All Symbols are stripped

# Deployment Postprocessing: true
# Strip Style: All Symbols (STRIP_STYLE=all) (default)
[~/Desktop] $ nm -a SampleApp/build/Release/SampleApp.app/Contents/MacOS/SampleApp
                 U _NSApplicationMain
                 U _OBJC_CLASS_$_NSObject
                 U _OBJC_METACLASS_$_NSObject
0000000100000000 A __mh_execute_header
                 U __objc_empty_cache
                 U __objc_empty_vtable
                 U _exit
                 U dyld_stub_binder
0000000005614542 - 00 0000   OPT radr://5614542

Ruby Script
To generate the statistics for the apps in /Applications, I wrote a short ruby script:

#!/usr/local/bin/ruby19

# analyze_apps.rb
# * Analyze dump from 'nm -a path_to_executable' to collect statistics: Name, Path, Lines, Archs, Files, Class Methods, Instance Methods

# Collect app paths.
apps = []
apps_folders = ["/Applications", "/Applications/iWork '08", "/Applications/iWork '09", "/Applications/Microsoft Office 2004", "/Applications/Microsoft Office 2008", "/Applications/Utilities"]
apps_folders.each do |app_folder|
  app_list = Dir.entries(app_folder)
  app_list.each do |app|
    apps << "#{app_folder}/#{app}"
  end
end

# Include source code?
PRINT_SOURCE_CODE = false

# Identify columns of data.
puts "Name, Path, Lines, Archs, Files, Class Methods, Instance Methods"

# Loop through apps.
apps.each do |app|
  # Check that the app exists.
  if Dir.exists?("#{app}") && app.include?(".app")
    app_array = app.split("/")
    app_name = app_array[app_array.count-1].split(".app")[0]
    exe_path = "#{app}/Contents/MacOS/#{app_name}"
    # Check that the executable exists.
    if File.exists?(exe_path)
      # Use 'nm -a path_to_executable' to dump symbol information.
      results = %x[nm -a "#{exe_path}"]

      # Get number of architectures supported. All other information is duplicated per architecture.
      arch_count = results.scan("(for architecture ").count
      arch_count = 1 if arch_count == 0

      # Get the number of lines, class methods, and instance methods per architecture.
      line_count = results.scan("\n").count / arch_count
      class_methods = results.scan(" t +[").count / arch_count
      instance_methods = results.scan(" t -[").count / arch_count

      # Loop through the results to more accurately count the source files (though not perfectly).
      source_lookup = []
      source_count = 0
      results.split("\n").each do |line|
        # Check that the line references "SO" or "SOL" but isn't repeated.
        if line.include?("SO ") && line.split("SO ").count > 1 && line.split("SO ")[1].include?(".m") && !source_lookup.include?(line.split("SO ")[1])
          file = line.split("SO ")[1]
          # Add file name to lookup.
          source_lookup << file
          # Increment counter.
          source_count = source_count + 1
          # Print out file name.
          puts "#{app_name}: #{file}" if PRINT_SOURCE_CODE
        elsif line.include?("SOL ") && line.split("SOL ").count > 1 && line.split("SOL ")[1].include?(".m") && !source_lookup.include?(line.split("SOL ")[1])
          file = line.split("SOL ")[1]
          # Add file name to lookup.
          source_lookup << file
          # Increment counter.
          source_count = source_count + 1
          # Print out file name.
          puts "#{app_name}: #{file}" if PRINT_SOURCE_CODE
        end
      end

      # Print out the application's stats.
      puts "#{app_name}, #{app}, #{line_count}, #{arch_count}, #{source_count}, #{class_methods}, #{instance_methods}"
    end
  end
end

- (void)viewDidAppear:(BOOL)animated {
	[preferencesListView reloadData]; // This call was necessary for the UITableView to keep its initial selection.
	[preferencesListView selectRowAtIndexPath:[NSIndexPath indexPathForRow:0 inSection:0] animated:NO scrollPosition:UITableViewScrollPositionNone];
	[self updateInterface];
	[super viewDidAppear:animated];
}

With a bit of help, I implemented a two-way pipe at a high-level and utilize that pipe to cleanly terminate the child process with a simple EOF signal. Apple’s BetterAuthorizationSample sample project touches on this communication pipe but does so at a low-level and only one-way. Matt Gallagher from Cocoa With Love also uses a pipe in invoking other processes when he creates an Open File Killer app, but the pipe is still only used one direction. Thanks to both those tutorials as well as a post from Caius Theory for helping me along. See my GitHub multicast_ping repo for the code in context.

Interface: setup the pipe to the tool

// Execute the command with privileges from SFAuthorizationView.
FILE *handle;
OSErr processError = AuthorizationExecuteWithPrivileges([[authView authorization] authorizationRef], [helperPath UTF8String],
														kAuthorizationFlagDefaults, (char *const *)argv, &handle);
free(argv);
if (processError != 0) {
	NSLog(@"helper tool failed (%d)", processError);
	return NO;
}

// Setup the two-way pipe.
helperHandle = [[NSFileHandle alloc] initWithFileDescriptor:fileno(handle)];
[helperHandle waitForDataInBackgroundAndNotify];
[[NSNotificationCenter defaultCenter] addObserver:self selector:@selector(handleTaskOutput:) name:NSFileHandleDataAvailableNotification object:helperHandle];

Interface: wait for data from the tool

- (void)handleTaskOutput:(NSNotification *)notification {
	// Get the new data.
	NSFileHandle *handle = (NSFileHandle *)[notification object];
	NSData *data = [handle availableData];
	if ([data length] > 0) {
		// Convert the data into a string.
		NSString *dataString = [[NSString alloc] initWithBytes:[data bytes] length:[data length] encoding:NSUTF8StringEncoding];
		if ([dataString isEqual:@"port in use"]) {
			// Tool failed.
			[self handleToolFailure];
			[dataString release];
			return;
		}

		// Process the string, expecting "address,port,message".
		NSArray *array = [dataString componentsSeparatedByString:@","];
		[dataString release];
		if ([array count] == 3) {
			// Get attributes.
			NSString *address = [array objectAtIndex:0];
			int port = [[array objectAtIndex:1] intValue];
			NSString *message = [array objectAtIndex:2];

			// Create new message.
			Message *newMessage = [[Message alloc] initWithAddress:address andPort:port andMessage:message];
			[messages addObject:newMessage];
			[newMessage release];

			// Update view.
			[listView reloadData];
		}

		// Prepare for more data.
		[handle waitForDataInBackgroundAndNotify];
	}
	else {
		// No data means tool failed.
		[self handleToolFailure];
	}
}

Interface: terminate the tool

// Close pipe to terminate the helper tool.
[helperHandle closeFile];
helperHandle = nil;

Tool: send data to the interface

// Write out the new message to the pipe.
NSString *line = [NSString stringWithFormat:@"%@,%d,%@", sentHost, sentPort, receivedMessage];
[(NSFileHandle *)[NSFileHandle fileHandleWithStandardOutput] writeData:[line dataUsingEncoding:NSUTF8StringEncoding]];

Tool: listen for terminate from the interface

// Wait for parent process to close the pipe. That will send the EOF signal.
[[NSFileHandle fileHandleWithStandardInput] readDataToEndOfFile];