# robots.txt # - a correct BNF is: # ROBOTS := [USERAGENT [DISALLOW]* ]* # USERAGENT := User-agent: *\n | AGENTNAME\n # DISALLOW := Disallow: /PATTERN\n # AGENTNAME := [a-zA-Z0-9_-.\%&$#=~()@ /]+ # PATTERN := [a-zA-Z0-9_-.\%&$#=~()@]*[/] # - in PATTERN, case is significant because filesystem names must be respected # - /pattern matches both /patternfile.extensions and /patternpath/files.html # - /pattern/ matches only /pattern/files.html # - User-agent: order is important. A catch-all entry must be the last one # User-agent: * Disallow: *