Security Archives - Shane Rutter Blog

Please double check any bots you wish to allow are not in the block list. This is a list of known bots, as well as default user agents.

I have had a scenario where a website was getting scraped by a number of bots, causing excessive resource usage on the server and disrupting sales and bad experience for real customers.

Here is a basic technique to stop those pesky bots, simply add the following to your .htaccess file. You may need to modify the exceptions at the top if you do require certain files to be accessed by bots. Currently it will allow access to robots.txt and acme-challenge for LetsEncrypt.

# BLOCK BAD BOTS START
RewriteEngine On
RewriteCond %{REQUEST_URI} !/robots.txt$
RewriteCond %{REQUEST_URI} !\.well-known/acme-challenge
RewriteCond %{HTTP_USER_AGENT} ^$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*EventMachine.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*NerdyBot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Typhoeus.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*archive.org_bot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*archive.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*github.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*chroot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Jorgee.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Go\ 1.1\ package.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Go-http-client.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Copyscape.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*semrushbot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*semrushbot-sa.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*JamesBOT.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*SEOkicks-Robot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*LinkpadBot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*getty.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*picscout.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*AppEngine.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Zend_Http_Client.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*BlackWidow.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*openlink.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*spbot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Nutch.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Jetbot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*WebVac.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Stanford.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*scooter.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*naver.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*dumbot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Hatena\ Antenna.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*grub.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*looksmart.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*WebZip.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*larbin.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*b2w/0.1.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Copernic.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*psbot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*NetMechanic.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*URL_Spider_Pro.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*CherryPicker.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*ExtractorPro.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*CopyRightCheck.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Crescent.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*CCBot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*SiteSnagger.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*ProWebWalker.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*CheeseBot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*LNSpiderguy.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*EmailCollector.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*EmailSiphon.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*WebBandit.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*EmailWolf.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*ia_archiver.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Alexibot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Teleport.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*MIIxpc.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Telesoft.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Website\ Quester.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*moget.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*WebStripper.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*WebSauger.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*WebCopier.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*NetAnts.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Mister\ PiX.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*WebAuto.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*TheNomad.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*WWW-Collector-E.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*RMA.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*libWeb/clsHTTP.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*asterias.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*httplib.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*turingos.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*spanner.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Harvest.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*InfoNaviRobot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Bullseye.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*WebBandit.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*NICErsPRO.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Microsoft\ URL\ Control.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*DittoSpyder.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Foobot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*WebmasterWorldForumBot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*SpankBot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*BotALot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*lwp-trivial.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*WebmasterWorld.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*BunnySlippers.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*URLy\ Warning.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*LinkWalker.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*cosmos.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*hloader.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*humanlinks.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*LinkextractorPro.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Offline\ Explorer.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Mata\ Hari.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*LexiBot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Web\ Image\ Collector.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*woobot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*The\ Intraformant.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*True_Robot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*BlowFish.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*SearchEngineWorld.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*JennyBot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*MIIxpc.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*BuiltBotTough.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*ProPowerBot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*BackDoorBot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*toCrawl/UrlDispatcher.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*WebEnhancer.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*suzuran.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*WebViewer.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*VCI.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Szukacz.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*QueryN.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Openfind.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Openbot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Webster.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*EroCrawler.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*LinkScan.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Keyword.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Kenjin.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Iron33.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Bookmark\ search\ tool.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*GetRight.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*FairAd\ Client.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Gaisbot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Aqua_Products.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Radiation\ Retriever\ 1.1.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Flaming\ AttackBot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Oracle\ Ultra\ Search.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*MSIECrawler.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*PerMan.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*searchpreview.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*sootle.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Enterprise_Search.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Bot\ mailto:craftbot@yahoo.com.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*ChinaClaw.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Custo.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*DISCo.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Download\ Demon.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*eCatch.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*EirGrabber.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*EmailSiphon.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*EmailWolf.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Express\ WebPictures.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*ExtractorPro.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*EyeNetIE.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*FlashGet.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*GetRight.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*GetWeb!.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Go!Zilla.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Go-Ahead-Got-It.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*GrabNet.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Grafula.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*HMView.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*HTTrack.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Image\ Stripper.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Image\ Sucker.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Indy\ Library.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*InterGET.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Internet\ Ninja.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*JetCar.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*JOC\ Web\ Spider.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*larbin.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*LeechFTP.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Mass\ Downloader.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*MIDown\ tool.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Mister\ PiX.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Navroad.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*NearSite.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*NetAnts.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*NetSpider.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Net\ Vampire.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*NetZIP.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Octopus.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Offline\ Explorer.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Offline\ Navigator.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*PageGrabber.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Papa\ Foto.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*pavuk.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*pcBrowser.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*RealDownload.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*ReGet.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*SiteSnagger.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*SmartDownload.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*SuperBot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*SuperHTTP.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Surfbot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*tAkeOut.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Teleport\ Pro.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*VoidEYE.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Web\ Image\ Collector.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Web\ Sucker.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*WebAuto.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*WebCopier.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*WebFetch.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*WebGo\ IS.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*WebLeacher.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*WebReaper.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*WebSauger.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*wesee.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Website\ eXtractor.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Website\ Quester.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*WebStripper.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*WebWhacker.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*WebZIP.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Wget.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Widow.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*WWWOFFLE.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Xaldon\ WebSpider.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Zeus.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Semrush.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*BecomeBot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Screaming.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Screaming\ FrogSEO.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*SEO.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*AhrefsBot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*MJ12bot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*rogerbot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*exabot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Xenu.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*dotbot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*gigabot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Twengabot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*htmlparser.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*libwww.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Python.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*perl.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*urllib.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*scan.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Curl.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*email.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*PycURL.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Python-urllib.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Pyth.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*PyQ.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*WebCollector.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*WebCopy.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*webcraw.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*webcraw.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*SurveyBot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*SurveyBot_IgnoreIP.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*BlekkoBot.*$ [NC]
RewriteRule ^.*.* http://www.google.com/ [L]
# BLOCK BAD BOTS END

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

# BLOCK BAD BOTS START

RewriteEngine On

RewriteCond %{REQUEST_URI} !/robots.txt$

RewriteCond %{REQUEST_URI} !\.well–known/acme–challenge

RewriteCond %{HTTP_USER_AGENT} ^$ [OR]

RewriteCond %{HTTP_USER_AGENT} ^.*EventMachine.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*NerdyBot.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Typhoeus.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*archive.org_bot.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*archive.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*github.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*chroot.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Jorgee.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Go\ 1.1\ package.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Go–http–client.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Copyscape.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*semrushbot.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*semrushbot–sa.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*JamesBOT.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*SEOkicks–Robot.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*LinkpadBot.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*getty.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*picscout.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*AppEngine.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Zend_Http_Client.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*BlackWidow.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*openlink.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*spbot.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Nutch.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Jetbot.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*WebVac.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Stanford.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*scooter.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*naver.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*dumbot.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Hatena\ Antenna.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*grub.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*looksmart.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*WebZip.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*larbin.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*b2w/0.1.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Copernic.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*psbot.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*NetMechanic.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*URL_Spider_Pro.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*CherryPicker.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*ExtractorPro.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*CopyRightCheck.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Crescent.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*CCBot.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*SiteSnagger.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*ProWebWalker.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*CheeseBot.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*LNSpiderguy.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*EmailCollector.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*EmailSiphon.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*WebBandit.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*EmailWolf.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*ia_archiver.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Alexibot.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Teleport.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*MIIxpc.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Telesoft.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Website\ Quester.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*moget.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*WebStripper.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*WebSauger.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*WebCopier.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*NetAnts.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Mister\ PiX.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*WebAuto.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*TheNomad.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*WWW–Collector–E.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*RMA.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*libWeb/clsHTTP.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*asterias.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*httplib.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*turingos.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*spanner.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Harvest.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*InfoNaviRobot.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Bullseye.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*WebBandit.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*NICErsPRO.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Microsoft\ URL\ Control.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*DittoSpyder.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Foobot.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*WebmasterWorldForumBot.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*SpankBot.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*BotALot.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*lwp–trivial.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*WebmasterWorld.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*BunnySlippers.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*URLy\ Warning.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*LinkWalker.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*cosmos.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*hloader.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*humanlinks.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*LinkextractorPro.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Offline\ Explorer.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Mata\ Hari.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*LexiBot.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Web\ Image\ Collector.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*woobot.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*The\ Intraformant.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*True_Robot.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*BlowFish.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*SearchEngineWorld.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*JennyBot.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*MIIxpc.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*BuiltBotTough.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*ProPowerBot.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*BackDoorBot.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*toCrawl/UrlDispatcher.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*WebEnhancer.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*suzuran.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*WebViewer.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*VCI.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Szukacz.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*QueryN.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Openfind.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Openbot.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Webster.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*EroCrawler.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*LinkScan.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Keyword.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Kenjin.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Iron33.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Bookmark\ search\ tool.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*GetRight.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*FairAd\ Client.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Gaisbot.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Aqua_Products.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Radiation\ Retriever\ 1.1.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Flaming\ AttackBot.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Oracle\ Ultra\ Search.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*MSIECrawler.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*PerMan.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*searchpreview.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*sootle.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Enterprise_Search.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Bot\ mailto:craftbot@yahoo.com.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*ChinaClaw.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Custo.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*DISCo.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Download\ Demon.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*eCatch.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*EirGrabber.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*EmailSiphon.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*EmailWolf.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Express\ WebPictures.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*ExtractorPro.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*EyeNetIE.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*FlashGet.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*GetRight.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*GetWeb!.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Go!Zilla.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Go–Ahead–Got–It.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*GrabNet.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Grafula.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*HMView.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*HTTrack.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Image\ Stripper.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Image\ Sucker.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Indy\ Library.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*InterGET.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Internet\ Ninja.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*JetCar.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*JOC\ Web\ Spider.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*larbin.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*LeechFTP.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Mass\ Downloader.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*MIDown\ tool.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Mister\ PiX.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Navroad.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*NearSite.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*NetAnts.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*NetSpider.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Net\ Vampire.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*NetZIP.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Octopus.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Offline\ Explorer.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Offline\ Navigator.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*PageGrabber.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Papa\ Foto.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*pavuk.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*pcBrowser.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*RealDownload.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*ReGet.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*SiteSnagger.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*SmartDownload.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*SuperBot.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*SuperHTTP.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Surfbot.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*tAkeOut.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Teleport\ Pro.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*VoidEYE.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Web\ Image\ Collector.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Web\ Sucker.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*WebAuto.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*WebCopier.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*WebFetch.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*WebGo\ IS.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*WebLeacher.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*WebReaper.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*WebSauger.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*wesee.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Website\ eXtractor.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Website\ Quester.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*WebStripper.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*WebWhacker.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*WebZIP.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Wget.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Widow.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*WWWOFFLE.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Xaldon\ WebSpider.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Zeus.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Semrush.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*BecomeBot.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Screaming.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Screaming\ FrogSEO.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*SEO.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*AhrefsBot.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*MJ12bot.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*rogerbot.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*exabot.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Xenu.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*dotbot.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*gigabot.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Twengabot.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*htmlparser.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*libwww.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Python.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*perl.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*urllib.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*scan.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Curl.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*email.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*PycURL.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Python–urllib.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*Pyth.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*PyQ.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*WebCollector.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*WebCopy.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*webcraw.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*SurveyBot.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*SurveyBot_IgnoreIP.*$ [NC,OR]

RewriteCond %{HTTP_USER_AGENT} ^.*BlekkoBot.*$ [NC]

RewriteRule ^.*.* http://www.google.com/ [L]

# BLOCK BAD BOTS END

Category: Security

htaccess: Block bad bots for security purposes

htaccess: HTTP to HTTPS redirect