From 9cc610cc12aaa290d901c4c033c2c0b1e753e592 Mon Sep 17 00:00:00 2001
From: Samuel Clay <samuel@ofbrooklyn.com>
Date: Tue, 4 Jan 2011 08:56:25 -0500
Subject: [PATCH] Adding robots.txt so only the homepage will be crawled.

---
 media/robots.txt | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 media/robots.txt

diff --git a/media/robots.txt b/media/robots.txt
new file mode 100644
index 000000000..a6121a671
--- /dev/null
+++ b/media/robots.txt
@@ -0,0 +1,8 @@
+User-agent: *
+Disallow: /reader
+Disallow: /rss_feeds
+Disallow: /classifier
+Disallow: /analyzer
+Disallow: /profile
+Disallow: /import
+Disallow: /media
\ No newline at end of file