# This filters configuration file determines whether URLs are included or ignored by the site content crawler.
# You can update the filters configuration file by using regular expressions to include or ignore values.

# Each non-comment, non-blank line contains a regular expression
# prefixed by '+' or '-'.  The first matching pattern in the file
# determines whether a URL is included or ignored.  If no pattern
# matches, the URL is ignored.

# skip file: ftp: and mailto: urls
-^(ftp|mailto|irc):

# skip URLs containing certain characters as probable queries, etc.
-[*!@#]

# skip URLs with slash-delimited segment that repeats 3+ times, to break loops
-.*(/.+?)/.*?\1/.*?\1/

# skip pages with logon required
-.*(webapp/wcs/stores/servlet/OrderCalculate).*
-.*(webapp/wcs/stores/servlet/LogonForm).*
-.*(webapp/wcs/stores/servlet/Ajax).*
-.*(webapp/wcs/stores/servlet/ClickInfo).*

-.*(Category).* 
-.*(Product).* 
-.*(TopCategories).* 
-.*(SearchDisplay).*

# accept only these hosts
+.*
