User-agent: * 
Disallow: /404/ 
Disallow: /404.shtml 
Disallow: /GeneratedItems/
Disallow: /cgi/
Disallow: /cgi-bin/ 

revisit: 5