summaryrefslogtreecommitdiff
path: root/lib/python2.7/robotparser.pyc
blob: ab3d853dc755d6845538ef481a057bff08abebd5 (plain)
ofshex dumpascii
0000 03 f3 0d 0a 2f 5c fc 55 63 00 00 00 00 00 00 00 00 03 00 00 00 40 00 00 00 73 7d 00 00 00 64 00 ..../\.Uc............@...s}...d.
0020 00 5a 00 00 64 01 00 64 02 00 6c 01 00 5a 01 00 64 01 00 64 02 00 6c 02 00 5a 02 00 64 03 00 67 .Z..d..d..l..Z..d..d..l..Z..d..g
0040 01 00 5a 03 00 64 03 00 64 0b 00 64 04 00 84 00 00 83 00 00 59 5a 04 00 64 05 00 64 0c 00 64 06 ..Z..d..d..d........YZ..d..d..d.
0060 00 84 00 00 83 00 00 59 5a 05 00 64 07 00 64 0d 00 64 08 00 84 00 00 83 00 00 59 5a 06 00 64 09 .......YZ..d..d..d........YZ..d.
0080 00 65 02 00 6a 07 00 66 01 00 64 0a 00 84 00 00 83 00 00 59 5a 08 00 64 02 00 53 28 0e 00 00 00 .e..j..f..d........YZ..d..S(....
00a0 73 3c 01 00 00 20 72 6f 62 6f 74 70 61 72 73 65 72 2e 70 79 0a 0a 20 20 20 20 43 6f 70 79 72 69 s<....robotparser.py......Copyri
00c0 67 68 74 20 28 43 29 20 32 30 30 30 20 20 42 61 73 74 69 61 6e 20 4b 6c 65 69 6e 65 69 64 61 6d ght.(C).2000..Bastian.Kleineidam
00e0 0a 0a 20 20 20 20 59 6f 75 20 63 61 6e 20 63 68 6f 6f 73 65 20 62 65 74 77 65 65 6e 20 74 77 6f ......You.can.choose.between.two
0100 20 6c 69 63 65 6e 73 65 73 20 77 68 65 6e 20 75 73 69 6e 67 20 74 68 69 73 20 70 61 63 6b 61 67 .licenses.when.using.this.packag
0120 65 3a 0a 20 20 20 20 31 29 20 47 4e 55 20 47 50 4c 76 32 0a 20 20 20 20 32 29 20 50 53 46 20 6c e:.....1).GNU.GPLv2.....2).PSF.l
0140 69 63 65 6e 73 65 20 66 6f 72 20 50 79 74 68 6f 6e 20 32 2e 32 0a 0a 20 20 20 20 54 68 65 20 72 icense.for.Python.2.2......The.r
0160 6f 62 6f 74 73 2e 74 78 74 20 45 78 63 6c 75 73 69 6f 6e 20 50 72 6f 74 6f 63 6f 6c 20 69 73 20 obots.txt.Exclusion.Protocol.is.
0180 69 6d 70 6c 65 6d 65 6e 74 65 64 20 61 73 20 73 70 65 63 69 66 69 65 64 20 69 6e 0a 20 20 20 20 implemented.as.specified.in.....
01a0 68 74 74 70 3a 2f 2f 69 6e 66 6f 2e 77 65 62 63 72 61 77 6c 65 72 2e 63 6f 6d 2f 6d 61 6b 2f 70 http://info.webcrawler.com/mak/p
01c0 72 6f 6a 65 63 74 73 2f 72 6f 62 6f 74 73 2f 6e 6f 72 6f 62 6f 74 73 2d 72 66 63 2e 68 74 6d 6c rojects/robots/norobots-rfc.html
01e0 0a 69 ff ff ff ff 4e 74 0f 00 00 00 52 6f 62 6f 74 46 69 6c 65 50 61 72 73 65 72 63 00 00 00 00 .i....Nt....RobotFileParserc....
0200 00 00 00 00 02 00 00 00 42 00 00 00 73 62 00 00 00 65 00 00 5a 01 00 64 00 00 5a 02 00 64 01 00 ........B...sb...e..Z..d..Z..d..
0220 64 02 00 84 01 00 5a 03 00 64 03 00 84 00 00 5a 04 00 64 04 00 84 00 00 5a 05 00 64 05 00 84 00 d.....Z..d.....Z..d.....Z..d....
0240 00 5a 06 00 64 06 00 84 00 00 5a 07 00 64 07 00 84 00 00 5a 08 00 64 08 00 84 00 00 5a 09 00 64 .Z..d.....Z..d.....Z..d.....Z..d
0260 09 00 84 00 00 5a 0a 00 64 0a 00 84 00 00 5a 0b 00 52 53 28 0b 00 00 00 73 73 00 00 00 20 54 68 .....Z..d.....Z..RS(....ss....Th
0280 69 73 20 63 6c 61 73 73 20 70 72 6f 76 69 64 65 73 20 61 20 73 65 74 20 6f 66 20 6d 65 74 68 6f is.class.provides.a.set.of.metho
02a0 64 73 20 74 6f 20 72 65 61 64 2c 20 70 61 72 73 65 20 61 6e 64 20 61 6e 73 77 65 72 0a 20 20 20 ds.to.read,.parse.and.answer....
02c0 20 71 75 65 73 74 69 6f 6e 73 20 61 62 6f 75 74 20 61 20 73 69 6e 67 6c 65 20 72 6f 62 6f 74 73 .questions.about.a.single.robots
02e0 2e 74 78 74 20 66 69 6c 65 2e 0a 0a 20 20 20 20 74 00 00 00 00 63 02 00 00 00 02 00 00 00 02 00 .txt.file.......t....c..........
0300 00 00 43 00 00 00 73 3e 00 00 00 67 00 00 7c 00 00 5f 00 00 64 00 00 7c 00 00 5f 02 00 74 03 00 ..C...s>...g..|.._..d..|.._..t..
0320 7c 00 00 5f 04 00 74 03 00 7c 00 00 5f 05 00 7c 00 00 6a 06 00 7c 01 00 83 01 00 01 64 01 00 7c |.._..t..|.._..|..j..|......d..|
0340 00 00 5f 07 00 64 00 00 53 28 02 00 00 00 4e 69 00 00 00 00 28 08 00 00 00 74 07 00 00 00 65 6e .._..d..S(....Ni....(....t....en
0360 74 72 69 65 73 74 04 00 00 00 4e 6f 6e 65 74 0d 00 00 00 64 65 66 61 75 6c 74 5f 65 6e 74 72 79 triest....Nonet....default_entry
0380 74 05 00 00 00 46 61 6c 73 65 74 0c 00 00 00 64 69 73 61 6c 6c 6f 77 5f 61 6c 6c 74 09 00 00 00 t....Falset....disallow_allt....
03a0 61 6c 6c 6f 77 5f 61 6c 6c 74 07 00 00 00 73 65 74 5f 75 72 6c 74 0c 00 00 00 6c 61 73 74 5f 63 allow_allt....set_urlt....last_c
03c0 68 65 63 6b 65 64 28 02 00 00 00 74 04 00 00 00 73 65 6c 66 74 03 00 00 00 75 72 6c 28 00 00 00 hecked(....t....selft....url(...
03e0 00 28 00 00 00 00 73 52 00 00 00 2f 62 75 69 6c 64 62 6f 74 2f 74 6d 70 2f 62 75 69 6c 64 68 6f .(....sR.../buildbot/tmp/buildho
0400 73 74 2f 69 6e 73 74 61 6c 6c 2f 70 72 65 62 75 69 6c 74 2f 6c 69 6e 75 78 2d 78 38 36 5f 36 34 st/install/prebuilt/linux-x86_64
0420 2f 6c 69 62 2f 70 79 74 68 6f 6e 32 2e 37 2f 72 6f 62 6f 74 70 61 72 73 65 72 2e 70 79 74 08 00 /lib/python2.7/robotparser.pyt..
0440 00 00 5f 5f 69 6e 69 74 5f 5f 18 00 00 00 73 0c 00 00 00 00 01 09 01 09 01 09 01 09 01 0d 01 63 ..__init__....s................c
0460 01 00 00 00 01 00 00 00 01 00 00 00 43 00 00 00 73 07 00 00 00 7c 00 00 6a 00 00 53 28 01 00 00 ............C...s....|..j..S(...
0480 00 73 b7 00 00 00 52 65 74 75 72 6e 73 20 74 68 65 20 74 69 6d 65 20 74 68 65 20 72 6f 62 6f 74 .s....Returns.the.time.the.robot
04a0 73 2e 74 78 74 20 66 69 6c 65 20 77 61 73 20 6c 61 73 74 20 66 65 74 63 68 65 64 2e 0a 0a 20 20 s.txt.file.was.last.fetched.....
04c0 20 20 20 20 20 20 54 68 69 73 20 69 73 20 75 73 65 66 75 6c 20 66 6f 72 20 6c 6f 6e 67 2d 72 75 ......This.is.useful.for.long-ru
04e0 6e 6e 69 6e 67 20 77 65 62 20 73 70 69 64 65 72 73 20 74 68 61 74 20 6e 65 65 64 20 74 6f 0a 20 nning.web.spiders.that.need.to..
0500 20 20 20 20 20 20 20 63 68 65 63 6b 20 66 6f 72 20 6e 65 77 20 72 6f 62 6f 74 73 2e 74 78 74 20 .......check.for.new.robots.txt.
0520 66 69 6c 65 73 20 70 65 72 69 6f 64 69 63 61 6c 6c 79 2e 0a 0a 20 20 20 20 20 20 20 20 28 01 00 files.periodically...........(..
0540 00 00 52 09 00 00 00 28 01 00 00 00 52 0a 00 00 00 28 00 00 00 00 28 00 00 00 00 73 52 00 00 00 ..R....(....R....(....(....sR...
0560 2f 62 75 69 6c 64 62 6f 74 2f 74 6d 70 2f 62 75 69 6c 64 68 6f 73 74 2f 69 6e 73 74 61 6c 6c 2f /buildbot/tmp/buildhost/install/
0580 70 72 65 62 75 69 6c 74 2f 6c 69 6e 75 78 2d 78 38 36 5f 36 34 2f 6c 69 62 2f 70 79 74 68 6f 6e prebuilt/linux-x86_64/lib/python
05a0 32 2e 37 2f 72 6f 62 6f 74 70 61 72 73 65 72 2e 70 79 74 05 00 00 00 6d 74 69 6d 65 20 00 00 00 2.7/robotparser.pyt....mtime....
05c0 73 02 00 00 00 00 07 63 01 00 00 00 02 00 00 00 02 00 00 00 43 00 00 00 73 1f 00 00 00 64 01 00 s......c............C...s....d..
05e0 64 02 00 6c 00 00 7d 01 00 7c 01 00 6a 00 00 83 00 00 7c 00 00 5f 01 00 64 02 00 53 28 03 00 00 d..l..}..|..j.....|.._..d..S(...
0600 00 73 59 00 00 00 53 65 74 73 20 74 68 65 20 74 69 6d 65 20 74 68 65 20 72 6f 62 6f 74 73 2e 74 .sY...Sets.the.time.the.robots.t
0620 78 74 20 66 69 6c 65 20 77 61 73 20 6c 61 73 74 20 66 65 74 63 68 65 64 20 74 6f 20 74 68 65 0a xt.file.was.last.fetched.to.the.
0640 20 20 20 20 20 20 20 20 63 75 72 72 65 6e 74 20 74 69 6d 65 2e 0a 0a 20 20 20 20 20 20 20 20 69 ........current.time...........i
0660 ff ff ff ff 4e 28 02 00 00 00 74 04 00 00 00 74 69 6d 65 52 09 00 00 00 28 02 00 00 00 52 0a 00 ....N(....t....timeR....(....R..
0680 00 00 52 0e 00 00 00 28 00 00 00 00 28 00 00 00 00 73 52 00 00 00 2f 62 75 69 6c 64 62 6f 74 2f ..R....(....(....sR.../buildbot/
06a0 74 6d 70 2f 62 75 69 6c 64 68 6f 73 74 2f 69 6e 73 74 61 6c 6c 2f 70 72 65 62 75 69 6c 74 2f 6c tmp/buildhost/install/prebuilt/l
06c0 69 6e 75 78 2d 78 38 36 5f 36 34 2f 6c 69 62 2f 70 79 74 68 6f 6e 32 2e 37 2f 72 6f 62 6f 74 70 inux-x86_64/lib/python2.7/robotp
06e0 61 72 73 65 72 2e 70 79 74 08 00 00 00 6d 6f 64 69 66 69 65 64 29 00 00 00 73 04 00 00 00 00 05 arser.pyt....modified)...s......
0700 0c 01 63 02 00 00 00 02 00 00 00 03 00 00 00 43 00 00 00 73 2f 00 00 00 7c 01 00 7c 00 00 5f 00 ..c............C...s/...|..|.._.
0720 00 74 01 00 6a 01 00 7c 01 00 83 01 00 64 01 00 64 02 00 21 5c 02 00 7c 00 00 5f 02 00 7c 00 00 .t..j..|.....d..d..!\..|.._..|..
0740 5f 03 00 64 03 00 53 28 04 00 00 00 73 2c 00 00 00 53 65 74 73 20 74 68 65 20 55 52 4c 20 72 65 _..d..S(....s,...Sets.the.URL.re
0760 66 65 72 72 69 6e 67 20 74 6f 20 61 20 72 6f 62 6f 74 73 2e 74 78 74 20 66 69 6c 65 2e 69 01 00 ferring.to.a.robots.txt.file.i..
0780 00 00 69 03 00 00 00 4e 28 04 00 00 00 52 0b 00 00 00 74 08 00 00 00 75 72 6c 70 61 72 73 65 74 ..i....N(....R....t....urlparset
07a0 04 00 00 00 68 6f 73 74 74 04 00 00 00 70 61 74 68 28 02 00 00 00 52 0a 00 00 00 52 0b 00 00 00 ....hostt....path(....R....R....
07c0 28 00 00 00 00 28 00 00 00 00 73 52 00 00 00 2f 62 75 69 6c 64 62 6f 74 2f 74 6d 70 2f 62 75 69 (....(....sR.../buildbot/tmp/bui
07e0 6c 64 68 6f 73 74 2f 69 6e 73 74 61 6c 6c 2f 70 72 65 62 75 69 6c 74 2f 6c 69 6e 75 78 2d 78 38 ldhost/install/prebuilt/linux-x8
0800 36 5f 36 34 2f 6c 69 62 2f 70 79 74 68 6f 6e 32 2e 37 2f 72 6f 62 6f 74 70 61 72 73 65 72 2e 70 6_64/lib/python2.7/robotparser.p
0820 79 52 08 00 00 00 31 00 00 00 73 04 00 00 00 00 02 09 01 63 01 00 00 00 05 00 00 00 03 00 00 00 yR....1...s........c............
0840 43 00 00 00 73 af 00 00 00 74 00 00 83 00 00 7d 01 00 7c 01 00 6a 01 00 7c 00 00 6a 02 00 83 01 C...s....t.....}..|..j..|..j....
0860 00 7d 02 00 67 00 00 7c 02 00 44 5d 12 00 7d 03 00 7c 03 00 6a 03 00 83 00 00 5e 02 00 71 22 00 .}..g..|..D]..}..|..j.....^..q".
0880 7d 04 00 7c 02 00 6a 04 00 83 00 00 01 7c 01 00 6a 05 00 7c 00 00 5f 05 00 7c 00 00 6a 05 00 64 }..|..j......|..j..|.._..|..j..d
08a0 06 00 6b 06 00 72 6b 00 74 06 00 7c 00 00 5f 07 00 6e 40 00 7c 00 00 6a 05 00 64 03 00 6b 05 00 ..k..rk.t..|.._..n@.|..j..d..k..
08c0 72 86 00 74 06 00 7c 00 00 5f 08 00 6e 25 00 7c 00 00 6a 05 00 64 04 00 6b 02 00 72 ab 00 7c 04 r..t..|.._..n%.|..j..d..k..r..|.
08e0 00 72 ab 00 7c 00 00 6a 09 00 7c 04 00 83 01 00 01 6e 00 00 64 05 00 53 28 07 00 00 00 73 34 00 .r..|..j..|......n..d..S(....s4.
0900 00 00 52 65 61 64 73 20 74 68 65 20 72 6f 62 6f 74 73 2e 74 78 74 20 55 52 4c 20 61 6e 64 20 66 ..Reads.the.robots.txt.URL.and.f
0920 65 65 64 73 20 69 74 20 74 6f 20 74 68 65 20 70 61 72 73 65 72 2e 69 91 01 00 00 69 93 01 00 00 eeds.it.to.the.parser.i....i....
0940 69 90 01 00 00 69 c8 00 00 00 4e 28 02 00 00 00 69 91 01 00 00 69 93 01 00 00 28 0a 00 00 00 74 i....i....N(....i....i....(....t
0960 09 00 00 00 55 52 4c 6f 70 65 6e 65 72 74 04 00 00 00 6f 70 65 6e 52 0b 00 00 00 74 05 00 00 00 ....URLopenert....openR....t....
0980 73 74 72 69 70 74 05 00 00 00 63 6c 6f 73 65 74 07 00 00 00 65 72 72 63 6f 64 65 74 04 00 00 00 stript....closet....errcodet....
09a0 54 72 75 65 52 06 00 00 00 52 07 00 00 00 74 05 00 00 00 70 61 72 73 65 28 05 00 00 00 52 0a 00 TrueR....R....t....parse(....R..
09c0 00 00 74 06 00 00 00 6f 70 65 6e 65 72 74 01 00 00 00 66 74 04 00 00 00 6c 69 6e 65 74 05 00 00 ..t....openert....ft....linet...
09e0 00 6c 69 6e 65 73 28 00 00 00 00 28 00 00 00 00 73 52 00 00 00 2f 62 75 69 6c 64 62 6f 74 2f 74 .lines(....(....sR.../buildbot/t
0a00 6d 70 2f 62 75 69 6c 64 68 6f 73 74 2f 69 6e 73 74 61 6c 6c 2f 70 72 65 62 75 69 6c 74 2f 6c 69 mp/buildhost/install/prebuilt/li
0a20 6e 75 78 2d 78 38 36 5f 36 34 2f 6c 69 62 2f 70 79 74 68 6f 6e 32 2e 37 2f 72 6f 62 6f 74 70 61 nux-x86_64/lib/python2.7/robotpa
0a40 72 73 65 72 2e 70 79 74 04 00 00 00 72 65 61 64 36 00 00 00 73 16 00 00 00 00 02 09 01 12 01 1f rser.pyt....read6...s...........
0a60 01 0a 01 0c 01 0f 01 0c 01 0f 01 0c 01 15 01 63 02 00 00 00 02 00 00 00 02 00 00 00 43 00 00 00 ...............c............C...
0a80 73 41 00 00 00 64 01 00 7c 01 00 6a 00 00 6b 06 00 72 2d 00 7c 00 00 6a 01 00 64 00 00 6b 08 00 sA...d..|..j..k..r-.|..j..d..k..
0aa0 72 3d 00 7c 01 00 7c 00 00 5f 01 00 71 3d 00 6e 10 00 7c 00 00 6a 03 00 6a 04 00 7c 01 00 83 01 r=.|..|.._..q=.n..|..j..j..|....
0ac0 00 01 64 00 00 53 28 02 00 00 00 4e 74 01 00 00 00 2a 28 05 00 00 00 74 0a 00 00 00 75 73 65 72 ..d..S(....Nt....*(....t....user
0ae0 61 67 65 6e 74 73 52 04 00 00 00 52 03 00 00 00 52 02 00 00 00 74 06 00 00 00 61 70 70 65 6e 64 agentsR....R....R....t....append
0b00 28 02 00 00 00 52 0a 00 00 00 74 05 00 00 00 65 6e 74 72 79 28 00 00 00 00 28 00 00 00 00 73 52 (....R....t....entry(....(....sR
0b20 00 00 00 2f 62 75 69 6c 64 62 6f 74 2f 74 6d 70 2f 62 75 69 6c 64 68 6f 73 74 2f 69 6e 73 74 61 .../buildbot/tmp/buildhost/insta
0b40 6c 6c 2f 70 72 65 62 75 69 6c 74 2f 6c 69 6e 75 78 2d 78 38 36 5f 36 34 2f 6c 69 62 2f 70 79 74 ll/prebuilt/linux-x86_64/lib/pyt
0b60 68 6f 6e 32 2e 37 2f 72 6f 62 6f 74 70 61 72 73 65 72 2e 70 79 74 0a 00 00 00 5f 61 64 64 5f 65 hon2.7/robotparser.pyt...._add_e
0b80 6e 74 72 79 44 00 00 00 73 08 00 00 00 00 01 0f 02 0f 02 0f 02 63 02 00 00 00 07 00 00 00 05 00 ntryD...s............c..........
0ba0 00 00 43 00 00 00 73 1c 02 00 00 64 01 00 7d 02 00 64 01 00 7d 03 00 74 00 00 83 00 00 7d 04 00 ..C...s....d..}..d..}..t.....}..
0bc0 78 e4 01 7c 01 00 44 5d dc 01 7d 05 00 7c 03 00 64 02 00 37 7d 03 00 7c 05 00 73 7e 00 7c 02 00 x..|..D]..}..|..d..7}..|..s~.|..
0be0 64 02 00 6b 02 00 72 50 00 74 00 00 83 00 00 7d 04 00 64 01 00 7d 02 00 71 7e 00 7c 02 00 64 03 d..k..rP.t.....}..d..}..q~.|..d.
0c00 00 6b 02 00 72 7e 00 7c 00 00 6a 01 00 7c 04 00 83 01 00 01 74 00 00 83 00 00 7d 04 00 64 01 00 .k..r~.|..j..|......t.....}..d..
0c20 7d 02 00 71 7e 00 6e 00 00 7c 05 00 6a 02 00 64 04 00 83 01 00 7d 06 00 7c 06 00 64 01 00 6b 05 }..q~.n..|..j..d.....}..|..d..k.
0c40 00 72 a6 00 7c 05 00 7c 06 00 20 7d 05 00 6e 00 00 7c 05 00 6a 03 00 83 00 00 7d 05 00 7c 05 00 .r..|..|...}..n..|..j.....}..|..
0c60 73 be 00 71 1c 00 6e 00 00 7c 05 00 6a 04 00 64 05 00 64 02 00 83 02 00 7d 05 00 74 05 00 7c 05 s..q..n..|..j..d..d.....}..t..|.
0c80 00 83 01 00 64 03 00 6b 02 00 72 1c 00 7c 05 00 64 01 00 19 6a 03 00 83 00 00 6a 06 00 83 00 00 ....d..k..r..|..d...j.....j.....
0ca0 7c 05 00 64 01 00 3c 74 07 00 6a 08 00 7c 05 00 64 02 00 19 6a 03 00 83 00 00 83 01 00 7c 05 00 |..d..<t..j..|..d...j........|..
0cc0 64 02 00 3c 7c 05 00 64 01 00 19 64 06 00 6b 02 00 72 6b 01 7c 02 00 64 03 00 6b 02 00 72 4e 01 d..<|..d...d..k..rk.|..d..k..rN.
0ce0 7c 00 00 6a 01 00 7c 04 00 83 01 00 01 74 00 00 83 00 00 7d 04 00 6e 00 00 7c 04 00 6a 09 00 6a |..j..|......t.....}..n..|..j..j
0d00 0a 00 7c 05 00 64 02 00 19 83 01 00 01 64 02 00 7d 02 00 71 f8 01 7c 05 00 64 01 00 19 64 07 00 ..|..d.......d..}..q..|..d...d..
0d20 6b 02 00 72 b0 01 7c 02 00 64 01 00 6b 03 00 72 f5 01 7c 04 00 6a 0b 00 6a 0a 00 74 0c 00 7c 05 k..r..|..d..k..r..|..j..j..t..|.
0d40 00 64 02 00 19 74 0d 00 83 02 00 83 01 00 01 64 03 00 7d 02 00 71 f5 01 71 f8 01 7c 05 00 64 01 .d...t.........d..}..q..q..|..d.
0d60 00 19 64 08 00 6b 02 00 72 f8 01 7c 02 00 64 01 00 6b 03 00 72 f5 01 7c 04 00 6a 0b 00 6a 0a 00 ..d..k..r..|..d..k..r..|..j..j..
0d80 74 0c 00 7c 05 00 64 02 00 19 74 0e 00 83 02 00 83 01 00 01 64 03 00 7d 02 00 71 f5 01 71 f8 01 t..|..d...t.........d..}..q..q..
0da0 71 1c 00 71 1c 00 57 7c 02 00 64 03 00 6b 02 00 72 18 02 7c 00 00 6a 01 00 7c 04 00 83 01 00 01 q..q..W|..d..k..r..|..j..|......
0dc0 6e 00 00 64 09 00 53 28 0a 00 00 00 73 90 00 00 00 70 61 72 73 65 20 74 68 65 20 69 6e 70 75 74 n..d..S(....s....parse.the.input
0de0 20 6c 69 6e 65 73 20 66 72 6f 6d 20 61 20 72 6f 62 6f 74 73 2e 74 78 74 20 66 69 6c 65 2e 0a 20 .lines.from.a.robots.txt.file...
0e00 20 20 20 20 20 20 20 20 20 20 57 65 20 61 6c 6c 6f 77 20 74 68 61 74 20 61 20 75 73 65 72 2d 61 ..........We.allow.that.a.user-a
0e20 67 65 6e 74 3a 20 6c 69 6e 65 20 69 73 20 6e 6f 74 20 70 72 65 63 65 64 65 64 20 62 79 0a 20 20 gent:.line.is.not.preceded.by...
0e40 20 20 20 20 20 20 20 20 20 6f 6e 65 20 6f 72 20 6d 6f 72 65 20 62 6c 61 6e 6b 20 6c 69 6e 65 73 .........one.or.more.blank.lines
0e60 2e 69 00 00 00 00 69 01 00 00 00 69 02 00 00 00 74 01 00 00 00 23 74 01 00 00 00 3a 73 0a 00 00 .i....i....i....t....#t....:s...
0e80 00 75 73 65 72 2d 61 67 65 6e 74 74 08 00 00 00 64 69 73 61 6c 6c 6f 77 74 05 00 00 00 61 6c 6c .user-agentt....disallowt....all
0ea0 6f 77 4e 28 0f 00 00 00 74 05 00 00 00 45 6e 74 72 79 52 23 00 00 00 74 04 00 00 00 66 69 6e 64 owN(....t....EntryR#...t....find
0ec0 52 15 00 00 00 74 05 00 00 00 73 70 6c 69 74 74 03 00 00 00 6c 65 6e 74 05 00 00 00 6c 6f 77 65 R....t....splitt....lent....lowe
0ee0 72 74 06 00 00 00 75 72 6c 6c 69 62 74 07 00 00 00 75 6e 71 75 6f 74 65 52 20 00 00 00 52 21 00 rt....urllibt....unquoteR....R!.
0f00 00 00 74 09 00 00 00 72 75 6c 65 6c 69 6e 65 73 74 08 00 00 00 52 75 6c 65 4c 69 6e 65 52 05 00 ..t....rulelinest....RuleLineR..
0f20 00 00 52 18 00 00 00 28 07 00 00 00 52 0a 00 00 00 52 1d 00 00 00 74 05 00 00 00 73 74 61 74 65 ..R....(....R....R....t....state
0f40 74 0a 00 00 00 6c 69 6e 65 6e 75 6d 62 65 72 52 22 00 00 00 52 1c 00 00 00 74 01 00 00 00 69 28 t....linenumberR"...R....t....i(
0f60 00 00 00 00 28 00 00 00 00 73 52 00 00 00 2f 62 75 69 6c 64 62 6f 74 2f 74 6d 70 2f 62 75 69 6c ....(....sR.../buildbot/tmp/buil
0f80 64 68 6f 73 74 2f 69 6e 73 74 61 6c 6c 2f 70 72 65 62 75 69 6c 74 2f 6c 69 6e 75 78 2d 78 38 36 dhost/install/prebuilt/linux-x86
0fa0 5f 36 34 2f 6c 69 62 2f 70 79 74 68 6f 6e 32 2e 37 2f 72 6f 62 6f 74 70 61 72 73 65 72 2e 70 79 _64/lib/python2.7/robotparser.py
0fc0 52 19 00 00 00 4d 00 00 00 73 4e 00 00 00 00 08 06 01 06 01 09 02 0d 01 0a 01 06 01 0c 01 09 01 R....M...sN.....................
0fe0 09 01 0c 01 0d 01 09 01 0c 02 0f 01 0c 01 0d 01 0c 01 06 01 06 01 12 01 12 01 1a 01 1d 01 10 01 ................................
1000 0c 01 0d 01 0c 01 14 01 09 01 10 01 0c 01 1d 01 0c 01 10 01 0c 01 1d 01 13 01 0c 01 63 03 00 00 ............................c...
1020 00 05 00 00 00 07 00 00 00 43 00 00 00 73 ca 00 00 00 7c 00 00 6a 00 00 72 0d 00 74 01 00 53 7c .........C...s....|..j..r..t..S|
1040 00 00 6a 02 00 72 1a 00 74 03 00 53 74 04 00 6a 04 00 74 05 00 6a 06 00 7c 02 00 83 01 00 83 01 ..j..r..t..St..j..t..j..|.......
1060 00 7d 03 00 74 04 00 6a 07 00 64 01 00 64 01 00 7c 03 00 6a 08 00 7c 03 00 6a 09 00 7c 03 00 6a .}..t..j..d..d..|..j..|..j..|..j
1080 0a 00 7c 03 00 6a 0b 00 66 06 00 83 01 00 7d 02 00 74 05 00 6a 0c 00 7c 02 00 83 01 00 7d 02 00 ..|..j..f.....}..t..j..|.....}..
10a0 7c 02 00 73 7d 00 64 02 00 7d 02 00 6e 00 00 78 2d 00 7c 00 00 6a 0d 00 44 5d 22 00 7d 04 00 7c |..s}.d..}..n..x-.|..j..D]".}..|
10c0 04 00 6a 0e 00 7c 01 00 83 01 00 72 87 00 7c 04 00 6a 0f 00 7c 02 00 83 01 00 53 71 87 00 57 7c ..j..|.....r..|..j..|.....Sq..W|
10e0 00 00 6a 10 00 72 c6 00 7c 00 00 6a 10 00 6a 0f 00 7c 02 00 83 01 00 53 74 03 00 53 28 03 00 00 ..j..r..|..j..j..|.....St..S(...
1100 00 73 3d 00 00 00 75 73 69 6e 67 20 74 68 65 20 70 61 72 73 65 64 20 72 6f 62 6f 74 73 2e 74 78 .s=...using.the.parsed.robots.tx
1120 74 20 64 65 63 69 64 65 20 69 66 20 75 73 65 72 61 67 65 6e 74 20 63 61 6e 20 66 65 74 63 68 20 t.decide.if.useragent.can.fetch.
1140 75 72 6c 52 01 00 00 00 74 01 00 00 00 2f 28 11 00 00 00 52 06 00 00 00 52 05 00 00 00 52 07 00 urlR....t..../(....R....R....R..
1160 00 00 52 18 00 00 00 52 10 00 00 00 52 2d 00 00 00 52 2e 00 00 00 74 0a 00 00 00 75 72 6c 75 6e ..R....R....R-...R....t....urlun
1180 70 61 72 73 65 52 12 00 00 00 74 06 00 00 00 70 61 72 61 6d 73 74 05 00 00 00 71 75 65 72 79 74 parseR....t....paramst....queryt
11a0 08 00 00 00 66 72 61 67 6d 65 6e 74 74 05 00 00 00 71 75 6f 74 65 52 02 00 00 00 74 0a 00 00 00 ....fragmentt....quoteR....t....
11c0 61 70 70 6c 69 65 73 5f 74 6f 74 09 00 00 00 61 6c 6c 6f 77 61 6e 63 65 52 04 00 00 00 28 05 00 applies_tot....allowanceR....(..
11e0 00 00 52 0a 00 00 00 74 09 00 00 00 75 73 65 72 61 67 65 6e 74 52 0b 00 00 00 74 0a 00 00 00 70 ..R....t....useragentR....t....p
1200 61 72 73 65 64 5f 75 72 6c 52 22 00 00 00 28 00 00 00 00 28 00 00 00 00 73 52 00 00 00 2f 62 75 arsed_urlR"...(....(....sR.../bu
1220 69 6c 64 62 6f 74 2f 74 6d 70 2f 62 75 69 6c 64 68 6f 73 74 2f 69 6e 73 74 61 6c 6c 2f 70 72 65 ildbot/tmp/buildhost/install/pre
1240 62 75 69 6c 74 2f 6c 69 6e 75 78 2d 78 38 36 5f 36 34 2f 6c 69 62 2f 70 79 74 68 6f 6e 32 2e 37 built/linux-x86_64/lib/python2.7
1260 2f 72 6f 62 6f 74 70 61 72 73 65 72 2e 70 79 74 09 00 00 00 63 61 6e 5f 66 65 74 63 68 80 00 00 /robotparser.pyt....can_fetch...
1280 00 73 20 00 00 00 00 02 09 01 04 01 09 01 04 03 18 01 12 01 1b 01 0f 01 06 01 09 01 10 01 0f 01 .s..............................
12a0 11 02 09 01 10 02 63 01 00 00 00 02 00 00 00 05 00 00 00 43 00 00 00 73 2d 00 00 00 64 01 00 6a ......c............C...s-...d..j
12c0 00 00 67 00 00 7c 00 00 6a 01 00 44 5d 16 00 7d 01 00 74 02 00 7c 01 00 83 01 00 64 02 00 17 5e ..g..|..j..D]..}..t..|.....d...^
12e0 02 00 71 10 00 83 01 00 53 28 03 00 00 00 4e 52 01 00 00 00 73 01 00 00 00 0a 28 03 00 00 00 74 ..q.....S(....NR....s.....(....t
1300 04 00 00 00 6a 6f 69 6e 52 02 00 00 00 74 03 00 00 00 73 74 72 28 02 00 00 00 52 0a 00 00 00 52 ....joinR....t....str(....R....R
1320 22 00 00 00 28 00 00 00 00 28 00 00 00 00 73 52 00 00 00 2f 62 75 69 6c 64 62 6f 74 2f 74 6d 70 "...(....(....sR.../buildbot/tmp
1340 2f 62 75 69 6c 64 68 6f 73 74 2f 69 6e 73 74 61 6c 6c 2f 70 72 65 62 75 69 6c 74 2f 6c 69 6e 75 /buildhost/install/prebuilt/linu
1360 78 2d 78 38 36 5f 36 34 2f 6c 69 62 2f 70 79 74 68 6f 6e 32 2e 37 2f 72 6f 62 6f 74 70 61 72 73 x-x86_64/lib/python2.7/robotpars
1380 65 72 2e 70 79 74 07 00 00 00 5f 5f 73 74 72 5f 5f 98 00 00 00 73 02 00 00 00 00 01 28 0c 00 00 er.pyt....__str__....s......(...
13a0 00 74 08 00 00 00 5f 5f 6e 61 6d 65 5f 5f 74 0a 00 00 00 5f 5f 6d 6f 64 75 6c 65 5f 5f 74 07 00 .t....__name__t....__module__t..
13c0 00 00 5f 5f 64 6f 63 5f 5f 52 0c 00 00 00 52 0d 00 00 00 52 0f 00 00 00 52 08 00 00 00 52 1e 00 ..__doc__R....R....R....R....R..
13e0 00 00 52 23 00 00 00 52 19 00 00 00 52 3e 00 00 00 52 41 00 00 00 28 00 00 00 00 28 00 00 00 00 ..R#...R....R>...RA...(....(....
1400 28 00 00 00 00 73 52 00 00 00 2f 62 75 69 6c 64 62 6f 74 2f 74 6d 70 2f 62 75 69 6c 64 68 6f 73 (....sR.../buildbot/tmp/buildhos
1420 74 2f 69 6e 73 74 61 6c 6c 2f 70 72 65 62 75 69 6c 74 2f 6c 69 6e 75 78 2d 78 38 36 5f 36 34 2f t/install/prebuilt/linux-x86_64/
1440 6c 69 62 2f 70 79 74 68 6f 6e 32 2e 37 2f 72 6f 62 6f 74 70 61 72 73 65 72 2e 70 79 52 00 00 00 lib/python2.7/robotparser.pyR...
1460 00 12 00 00 00 73 14 00 00 00 06 04 06 02 0c 08 09 09 09 08 09 05 09 0e 09 09 09 33 09 18 52 30 .....s.....................3..R0
1480 00 00 00 63 00 00 00 00 00 00 00 00 01 00 00 00 42 00 00 00 73 29 00 00 00 65 00 00 5a 01 00 64 ...c............B...s)...e..Z..d
14a0 00 00 5a 02 00 64 01 00 84 00 00 5a 03 00 64 02 00 84 00 00 5a 04 00 64 03 00 84 00 00 5a 05 00 ..Z..d.....Z..d.....Z..d.....Z..
14c0 52 53 28 04 00 00 00 73 6f 00 00 00 41 20 72 75 6c 65 20 6c 69 6e 65 20 69 73 20 61 20 73 69 6e RS(....so...A.rule.line.is.a.sin
14e0 67 6c 65 20 22 41 6c 6c 6f 77 3a 22 20 28 61 6c 6c 6f 77 61 6e 63 65 3d 3d 54 72 75 65 29 20 6f gle."Allow:".(allowance==True).o
1500 72 20 22 44 69 73 61 6c 6c 6f 77 3a 22 0a 20 20 20 20 20 20 20 28 61 6c 6c 6f 77 61 6e 63 65 3d r."Disallow:"........(allowance=
1520 3d 46 61 6c 73 65 29 20 66 6f 6c 6c 6f 77 65 64 20 62 79 20 61 20 70 61 74 68 2e 63 03 00 00 00 =False).followed.by.a.path.c....
1540 03 00 00 00 03 00 00 00 43 00 00 00 73 3b 00 00 00 7c 01 00 64 01 00 6b 02 00 72 1c 00 7c 02 00 ........C...s;...|..d..k..r..|..
1560 0c 72 1c 00 74 00 00 7d 02 00 6e 00 00 74 01 00 6a 02 00 7c 01 00 83 01 00 7c 00 00 5f 03 00 7c .r..t..}..n..t..j..|.....|.._..|
1580 02 00 7c 00 00 5f 04 00 64 00 00 53 28 02 00 00 00 4e 52 01 00 00 00 28 05 00 00 00 52 18 00 00 ..|.._..d..S(....NR....(....R...
15a0 00 52 2d 00 00 00 52 39 00 00 00 52 12 00 00 00 52 3b 00 00 00 28 03 00 00 00 52 0a 00 00 00 52 .R-...R9...R....R;...(....R....R
15c0 12 00 00 00 52 3b 00 00 00 28 00 00 00 00 28 00 00 00 00 73 52 00 00 00 2f 62 75 69 6c 64 62 6f ....R;...(....(....sR.../buildbo
15e0 74 2f 74 6d 70 2f 62 75 69 6c 64 68 6f 73 74 2f 69 6e 73 74 61 6c 6c 2f 70 72 65 62 75 69 6c 74 t/tmp/buildhost/install/prebuilt
1600 2f 6c 69 6e 75 78 2d 78 38 36 5f 36 34 2f 6c 69 62 2f 70 79 74 68 6f 6e 32 2e 37 2f 72 6f 62 6f /linux-x86_64/lib/python2.7/robo
1620 74 70 61 72 73 65 72 2e 70 79 52 0c 00 00 00 9f 00 00 00 73 08 00 00 00 00 01 13 02 09 01 12 01 tparser.pyR........s............
1640 63 02 00 00 00 02 00 00 00 03 00 00 00 43 00 00 00 73 1f 00 00 00 7c 00 00 6a 00 00 64 01 00 6b c............C...s....|..j..d..k
1660 02 00 70 1e 00 7c 01 00 6a 01 00 7c 00 00 6a 00 00 83 01 00 53 28 02 00 00 00 4e 52 1f 00 00 00 ..p..|..j..|..j.....S(....NR....
1680 28 02 00 00 00 52 12 00 00 00 74 0a 00 00 00 73 74 61 72 74 73 77 69 74 68 28 02 00 00 00 52 0a (....R....t....startswith(....R.
16a0 00 00 00 74 08 00 00 00 66 69 6c 65 6e 61 6d 65 28 00 00 00 00 28 00 00 00 00 73 52 00 00 00 2f ...t....filename(....(....sR.../
16c0 62 75 69 6c 64 62 6f 74 2f 74 6d 70 2f 62 75 69 6c 64 68 6f 73 74 2f 69 6e 73 74 61 6c 6c 2f 70 buildbot/tmp/buildhost/install/p
16e0 72 65 62 75 69 6c 74 2f 6c 69 6e 75 78 2d 78 38 36 5f 36 34 2f 6c 69 62 2f 70 79 74 68 6f 6e 32 rebuilt/linux-x86_64/lib/python2
1700 2e 37 2f 72 6f 62 6f 74 70 61 72 73 65 72 2e 70 79 52 3a 00 00 00 a6 00 00 00 73 02 00 00 00 00 .7/robotparser.pyR:.......s.....
1720 01 63 01 00 00 00 01 00 00 00 04 00 00 00 43 00 00 00 73 1e 00 00 00 7c 00 00 6a 00 00 72 0f 00 .c............C...s....|..j..r..
1740 64 01 00 70 12 00 64 02 00 64 03 00 17 7c 00 00 6a 01 00 17 53 28 04 00 00 00 4e 74 05 00 00 00 d..p..d..d...|..j...S(....Nt....
1760 41 6c 6c 6f 77 74 08 00 00 00 44 69 73 61 6c 6c 6f 77 73 02 00 00 00 3a 20 28 02 00 00 00 52 3b Allowt....Disallows....:.(....R;
1780 00 00 00 52 12 00 00 00 28 01 00 00 00 52 0a 00 00 00 28 00 00 00 00 28 00 00 00 00 73 52 00 00 ...R....(....R....(....(....sR..
17a0 00 2f 62 75 69 6c 64 62 6f 74 2f 74 6d 70 2f 62 75 69 6c 64 68 6f 73 74 2f 69 6e 73 74 61 6c 6c ./buildbot/tmp/buildhost/install
17c0 2f 70 72 65 62 75 69 6c 74 2f 6c 69 6e 75 78 2d 78 38 36 5f 36 34 2f 6c 69 62 2f 70 79 74 68 6f /prebuilt/linux-x86_64/lib/pytho
17e0 6e 32 2e 37 2f 72 6f 62 6f 74 70 61 72 73 65 72 2e 70 79 52 41 00 00 00 a9 00 00 00 73 02 00 00 n2.7/robotparser.pyRA.......s...
1800 00 00 01 28 06 00 00 00 52 42 00 00 00 52 43 00 00 00 52 44 00 00 00 52 0c 00 00 00 52 3a 00 00 ...(....RB...RC...RD...R....R:..
1820 00 52 41 00 00 00 28 00 00 00 00 28 00 00 00 00 28 00 00 00 00 73 52 00 00 00 2f 62 75 69 6c 64 .RA...(....(....(....sR.../build
1840 62 6f 74 2f 74 6d 70 2f 62 75 69 6c 64 68 6f 73 74 2f 69 6e 73 74 61 6c 6c 2f 70 72 65 62 75 69 bot/tmp/buildhost/install/prebui
1860 6c 74 2f 6c 69 6e 75 78 2d 78 38 36 5f 36 34 2f 6c 69 62 2f 70 79 74 68 6f 6e 32 2e 37 2f 72 6f lt/linux-x86_64/lib/python2.7/ro
1880 62 6f 74 70 61 72 73 65 72 2e 70 79 52 30 00 00 00 9c 00 00 00 73 08 00 00 00 06 02 06 01 09 07 botparser.pyR0.......s..........
18a0 09 03 52 28 00 00 00 63 00 00 00 00 00 00 00 00 01 00 00 00 42 00 00 00 73 32 00 00 00 65 00 00 ..R(...c............B...s2...e..
18c0 5a 01 00 64 00 00 5a 02 00 64 01 00 84 00 00 5a 03 00 64 02 00 84 00 00 5a 04 00 64 03 00 84 00 Z..d..Z..d.....Z..d.....Z..d....
18e0 00 5a 05 00 64 04 00 84 00 00 5a 06 00 52 53 28 05 00 00 00 73 3f 00 00 00 41 6e 20 65 6e 74 72 .Z..d.....Z..RS(....s?...An.entr
1900 79 20 68 61 73 20 6f 6e 65 20 6f 72 20 6d 6f 72 65 20 75 73 65 72 2d 61 67 65 6e 74 73 20 61 6e y.has.one.or.more.user-agents.an
1920 64 20 7a 65 72 6f 20 6f 72 20 6d 6f 72 65 20 72 75 6c 65 6c 69 6e 65 73 63 01 00 00 00 01 00 00 d.zero.or.more.rulelinesc.......
1940 00 02 00 00 00 43 00 00 00 73 16 00 00 00 67 00 00 7c 00 00 5f 00 00 67 00 00 7c 00 00 5f 01 00 .....C...s....g..|.._..g..|.._..
1960 64 00 00 53 28 01 00 00 00 4e 28 02 00 00 00 52 20 00 00 00 52 2f 00 00 00 28 01 00 00 00 52 0a d..S(....N(....R....R/...(....R.
1980 00 00 00 28 00 00 00 00 28 00 00 00 00 73 52 00 00 00 2f 62 75 69 6c 64 62 6f 74 2f 74 6d 70 2f ...(....(....sR.../buildbot/tmp/
19a0 62 75 69 6c 64 68 6f 73 74 2f 69 6e 73 74 61 6c 6c 2f 70 72 65 62 75 69 6c 74 2f 6c 69 6e 75 78 buildhost/install/prebuilt/linux
19c0 2d 78 38 36 5f 36 34 2f 6c 69 62 2f 70 79 74 68 6f 6e 32 2e 37 2f 72 6f 62 6f 74 70 61 72 73 65 -x86_64/lib/python2.7/robotparse
19e0 72 2e 70 79 52 0c 00 00 00 af 00 00 00 73 04 00 00 00 00 01 09 01 63 01 00 00 00 04 00 00 00 05 r.pyR........s........c.........
1a00 00 00 00 43 00 00 00 73 6a 00 00 00 67 00 00 7d 01 00 78 27 00 7c 00 00 6a 00 00 44 5d 1c 00 7d ...C...sj...g..}..x'.|..j..D]..}
1a20 02 00 7c 01 00 6a 01 00 64 01 00 7c 02 00 64 02 00 67 03 00 83 01 00 01 71 10 00 57 78 2a 00 7c ..|..j..d..|..d..g......q..Wx*.|
1a40 00 00 6a 02 00 44 5d 1f 00 7d 03 00 7c 01 00 6a 01 00 74 03 00 7c 03 00 83 01 00 64 02 00 67 02 ..j..D]..}..|..j..t..|.....d..g.
1a60 00 83 01 00 01 71 3a 00 57 64 03 00 6a 04 00 7c 01 00 83 01 00 53 28 04 00 00 00 4e 73 0c 00 00 .....q:.Wd..j..|.....S(....Ns...
1a80 00 55 73 65 72 2d 61 67 65 6e 74 3a 20 73 01 00 00 00 0a 52 01 00 00 00 28 05 00 00 00 52 20 00 .User-agent:.s.....R....(....R..
1aa0 00 00 74 06 00 00 00 65 78 74 65 6e 64 52 2f 00 00 00 52 40 00 00 00 52 3f 00 00 00 28 04 00 00 ..t....extendR/...R@...R?...(...
1ac0 00 52 0a 00 00 00 74 03 00 00 00 72 65 74 74 05 00 00 00 61 67 65 6e 74 52 1c 00 00 00 28 00 00 .R....t....rett....agentR....(..
1ae0 00 00 28 00 00 00 00 73 52 00 00 00 2f 62 75 69 6c 64 62 6f 74 2f 74 6d 70 2f 62 75 69 6c 64 68 ..(....sR.../buildbot/tmp/buildh
1b00 6f 73 74 2f 69 6e 73 74 61 6c 6c 2f 70 72 65 62 75 69 6c 74 2f 6c 69 6e 75 78 2d 78 38 36 5f 36 ost/install/prebuilt/linux-x86_6
1b20 34 2f 6c 69 62 2f 70 79 74 68 6f 6e 32 2e 37 2f 72 6f 62 6f 74 70 61 72 73 65 72 2e 70 79 52 41 4/lib/python2.7/robotparser.pyRA
1b40 00 00 00 b3 00 00 00 73 0c 00 00 00 00 01 06 01 10 01 1a 01 10 01 1d 01 63 02 00 00 00 03 00 00 .......s................c.......
1b60 00 03 00 00 00 43 00 00 00 73 5d 00 00 00 7c 01 00 6a 00 00 64 01 00 83 01 00 64 02 00 19 6a 01 .....C...s]...|..j..d.....d...j.
1b80 00 83 00 00 7d 01 00 78 3d 00 7c 00 00 6a 02 00 44 5d 32 00 7d 02 00 7c 02 00 64 03 00 6b 02 00 ....}..x=.|..j..D]2.}..|..d..k..
1ba0 72 39 00 74 03 00 53 7c 02 00 6a 01 00 83 00 00 7d 02 00 7c 02 00 7c 01 00 6b 06 00 72 23 00 74 r9.t..S|..j.....}..|..|..k..r#.t
1bc0 03 00 53 71 23 00 57 74 04 00 53 28 04 00 00 00 73 32 00 00 00 63 68 65 63 6b 20 69 66 20 74 68 ..Sq#.Wt..S(....s2...check.if.th
1be0 69 73 20 65 6e 74 72 79 20 61 70 70 6c 69 65 73 20 74 6f 20 74 68 65 20 73 70 65 63 69 66 69 65 is.entry.applies.to.the.specifie
1c00 64 20 61 67 65 6e 74 52 34 00 00 00 69 00 00 00 00 52 1f 00 00 00 28 05 00 00 00 52 2a 00 00 00 d.agentR4...i....R....(....R*...
1c20 52 2c 00 00 00 52 20 00 00 00 52 18 00 00 00 52 05 00 00 00 28 03 00 00 00 52 0a 00 00 00 52 3c R,...R....R....R....(....R....R<
1c40 00 00 00 52 4b 00 00 00 28 00 00 00 00 28 00 00 00 00 73 52 00 00 00 2f 62 75 69 6c 64 62 6f 74 ...RK...(....(....sR.../buildbot
1c60 2f 74 6d 70 2f 62 75 69 6c 64 68 6f 73 74 2f 69 6e 73 74 61 6c 6c 2f 70 72 65 62 75 69 6c 74 2f /tmp/buildhost/install/prebuilt/
1c80 6c 69 6e 75 78 2d 78 38 36 5f 36 34 2f 6c 69 62 2f 70 79 74 68 6f 6e 32 2e 37 2f 72 6f 62 6f 74 linux-x86_64/lib/python2.7/robot
1ca0 70 61 72 73 65 72 2e 70 79 52 3a 00 00 00 bb 00 00 00 73 10 00 00 00 00 03 19 01 10 01 0c 02 04 parser.pyR:.......s.............
1cc0 01 0c 01 0c 01 08 01 63 02 00 00 00 03 00 00 00 03 00 00 00 43 00 00 00 73 2e 00 00 00 78 27 00 .......c............C...s....x'.
1ce0 7c 00 00 6a 00 00 44 5d 1c 00 7d 02 00 7c 02 00 6a 01 00 7c 01 00 83 01 00 72 0a 00 7c 02 00 6a |..j..D]..}..|..j..|.....r..|..j
1d00 02 00 53 71 0a 00 57 74 03 00 53 28 01 00 00 00 73 5a 00 00 00 50 72 65 63 6f 6e 64 69 74 69 6f ..Sq..Wt..S(....sZ...Preconditio
1d20 6e 73 3a 0a 20 20 20 20 20 20 20 20 2d 20 6f 75 72 20 61 67 65 6e 74 20 61 70 70 6c 69 65 73 20 ns:.........-.our.agent.applies.
1d40 74 6f 20 74 68 69 73 20 65 6e 74 72 79 0a 20 20 20 20 20 20 20 20 2d 20 66 69 6c 65 6e 61 6d 65 to.this.entry.........-.filename
1d60 20 69 73 20 55 52 4c 20 64 65 63 6f 64 65 64 28 04 00 00 00 52 2f 00 00 00 52 3a 00 00 00 52 3b .is.URL.decoded(....R/...R:...R;
1d80 00 00 00 52 18 00 00 00 28 03 00 00 00 52 0a 00 00 00 52 46 00 00 00 52 1c 00 00 00 28 00 00 00 ...R....(....R....RF...R....(...
1da0 00 28 00 00 00 00 73 52 00 00 00 2f 62 75 69 6c 64 62 6f 74 2f 74 6d 70 2f 62 75 69 6c 64 68 6f .(....sR.../buildbot/tmp/buildho
1dc0 73 74 2f 69 6e 73 74 61 6c 6c 2f 70 72 65 62 75 69 6c 74 2f 6c 69 6e 75 78 2d 78 38 36 5f 36 34 st/install/prebuilt/linux-x86_64
1de0 2f 6c 69 62 2f 70 79 74 68 6f 6e 32 2e 37 2f 72 6f 62 6f 74 70 61 72 73 65 72 2e 70 79 52 3b 00 /lib/python2.7/robotparser.pyR;.
1e00 00 00 c8 00 00 00 73 08 00 00 00 00 04 10 01 0f 01 0b 01 28 07 00 00 00 52 42 00 00 00 52 43 00 ......s............(....RB...RC.
1e20 00 00 52 44 00 00 00 52 0c 00 00 00 52 41 00 00 00 52 3a 00 00 00 52 3b 00 00 00 28 00 00 00 00 ..RD...R....RA...R:...R;...(....
1e40 28 00 00 00 00 28 00 00 00 00 73 52 00 00 00 2f 62 75 69 6c 64 62 6f 74 2f 74 6d 70 2f 62 75 69 (....(....sR.../buildbot/tmp/bui
1e60 6c 64 68 6f 73 74 2f 69 6e 73 74 61 6c 6c 2f 70 72 65 62 75 69 6c 74 2f 6c 69 6e 75 78 2d 78 38 ldhost/install/prebuilt/linux-x8
1e80 36 5f 36 34 2f 6c 69 62 2f 70 79 74 68 6f 6e 32 2e 37 2f 72 6f 62 6f 74 70 61 72 73 65 72 2e 70 6_64/lib/python2.7/robotparser.p
1ea0 79 52 28 00 00 00 ad 00 00 00 73 0a 00 00 00 06 01 06 01 09 04 09 08 09 0d 52 13 00 00 00 63 00 yR(.......s..............R....c.
1ec0 00 00 00 00 00 00 00 01 00 00 00 42 00 00 00 73 23 00 00 00 65 00 00 5a 01 00 64 00 00 84 00 00 ...........B...s#...e..Z..d.....
1ee0 5a 02 00 64 01 00 84 00 00 5a 03 00 64 02 00 84 00 00 5a 04 00 52 53 28 03 00 00 00 63 01 00 00 Z..d.....Z..d.....Z..RS(....c...
1f00 00 02 00 00 00 03 00 00 00 47 00 00 00 73 20 00 00 00 74 00 00 6a 01 00 6a 02 00 7c 00 00 7c 01 .........G...s....t..j..j..|..|.
1f20 00 8c 01 00 01 64 01 00 7c 00 00 5f 03 00 64 00 00 53 28 02 00 00 00 4e 69 c8 00 00 00 28 04 00 .....d..|.._..d..S(....Ni....(..
1f40 00 00 52 2d 00 00 00 74 0e 00 00 00 46 61 6e 63 79 55 52 4c 6f 70 65 6e 65 72 52 0c 00 00 00 52 ..R-...t....FancyURLopenerR....R
1f60 17 00 00 00 28 02 00 00 00 52 0a 00 00 00 74 04 00 00 00 61 72 67 73 28 00 00 00 00 28 00 00 00 ....(....R....t....args(....(...
1f80 00 73 52 00 00 00 2f 62 75 69 6c 64 62 6f 74 2f 74 6d 70 2f 62 75 69 6c 64 68 6f 73 74 2f 69 6e .sR.../buildbot/tmp/buildhost/in
1fa0 73 74 61 6c 6c 2f 70 72 65 62 75 69 6c 74 2f 6c 69 6e 75 78 2d 78 38 36 5f 36 34 2f 6c 69 62 2f stall/prebuilt/linux-x86_64/lib/
1fc0 70 79 74 68 6f 6e 32 2e 37 2f 72 6f 62 6f 74 70 61 72 73 65 72 2e 70 79 52 0c 00 00 00 d2 00 00 python2.7/robotparser.pyR.......
1fe0 00 73 04 00 00 00 00 01 13 01 63 03 00 00 00 03 00 00 00 02 00 00 00 43 00 00 00 73 04 00 00 00 .s........c............C...s....
2000 64 01 00 53 28 02 00 00 00 4e 28 02 00 00 00 4e 4e 28 01 00 00 00 52 03 00 00 00 28 03 00 00 00 d..S(....N(....NN(....R....(....
2020 52 0a 00 00 00 52 11 00 00 00 74 05 00 00 00 72 65 61 6c 6d 28 00 00 00 00 28 00 00 00 00 73 52 R....R....t....realm(....(....sR
2040 00 00 00 2f 62 75 69 6c 64 62 6f 74 2f 74 6d 70 2f 62 75 69 6c 64 68 6f 73 74 2f 69 6e 73 74 61 .../buildbot/tmp/buildhost/insta
2060 6c 6c 2f 70 72 65 62 75 69 6c 74 2f 6c 69 6e 75 78 2d 78 38 36 5f 36 34 2f 6c 69 62 2f 70 79 74 ll/prebuilt/linux-x86_64/lib/pyt
2080 68 6f 6e 32 2e 37 2f 72 6f 62 6f 74 70 61 72 73 65 72 2e 70 79 74 12 00 00 00 70 72 6f 6d 70 74 hon2.7/robotparser.pyt....prompt
20a0 5f 75 73 65 72 5f 70 61 73 73 77 64 d6 00 00 00 73 02 00 00 00 00 03 63 06 00 00 00 06 00 00 00 _user_passwd....s......c........
20c0 07 00 00 00 43 00 00 00 73 28 00 00 00 7c 03 00 7c 00 00 5f 00 00 74 01 00 6a 02 00 6a 03 00 7c ....C...s(...|..|.._..t..j..j..|
20e0 00 00 7c 01 00 7c 02 00 7c 03 00 7c 04 00 7c 05 00 83 06 00 53 28 01 00 00 00 4e 28 04 00 00 00 ..|..|..|..|..|.....S(....N(....
2100 52 17 00 00 00 52 2d 00 00 00 52 4c 00 00 00 74 12 00 00 00 68 74 74 70 5f 65 72 72 6f 72 5f 64 R....R-...RL...t....http_error_d
2120 65 66 61 75 6c 74 28 06 00 00 00 52 0a 00 00 00 52 0b 00 00 00 74 02 00 00 00 66 70 52 17 00 00 efault(....R....R....t....fpR...
2140 00 74 06 00 00 00 65 72 72 6d 73 67 74 07 00 00 00 68 65 61 64 65 72 73 28 00 00 00 00 28 00 00 .t....errmsgt....headers(....(..
2160 00 00 73 52 00 00 00 2f 62 75 69 6c 64 62 6f 74 2f 74 6d 70 2f 62 75 69 6c 64 68 6f 73 74 2f 69 ..sR.../buildbot/tmp/buildhost/i
2180 6e 73 74 61 6c 6c 2f 70 72 65 62 75 69 6c 74 2f 6c 69 6e 75 78 2d 78 38 36 5f 36 34 2f 6c 69 62 nstall/prebuilt/linux-x86_64/lib
21a0 2f 70 79 74 68 6f 6e 32 2e 37 2f 72 6f 62 6f 74 70 61 72 73 65 72 2e 70 79 52 50 00 00 00 db 00 /python2.7/robotparser.pyRP.....
21c0 00 00 73 06 00 00 00 00 01 09 01 15 01 28 05 00 00 00 52 42 00 00 00 52 43 00 00 00 52 0c 00 00 ..s..........(....RB...RC...R...
21e0 00 52 4f 00 00 00 52 50 00 00 00 28 00 00 00 00 28 00 00 00 00 28 00 00 00 00 73 52 00 00 00 2f .RO...RP...(....(....(....sR.../
2200 62 75 69 6c 64 62 6f 74 2f 74 6d 70 2f 62 75 69 6c 64 68 6f 73 74 2f 69 6e 73 74 61 6c 6c 2f 70 buildbot/tmp/buildhost/install/p
2220 72 65 62 75 69 6c 74 2f 6c 69 6e 75 78 2d 78 38 36 5f 36 34 2f 6c 69 62 2f 70 79 74 68 6f 6e 32 rebuilt/linux-x86_64/lib/python2
2240 2e 37 2f 72 6f 62 6f 74 70 61 72 73 65 72 2e 70 79 52 13 00 00 00 d1 00 00 00 73 06 00 00 00 06 .7/robotparser.pyR........s.....
2260 01 09 04 09 05 28 00 00 00 00 28 00 00 00 00 28 00 00 00 00 28 09 00 00 00 52 44 00 00 00 52 10 .....(....(....(....(....RD...R.
2280 00 00 00 52 2d 00 00 00 74 07 00 00 00 5f 5f 61 6c 6c 5f 5f 52 00 00 00 00 52 30 00 00 00 52 28 ...R-...t....__all__R....R0...R(
22a0 00 00 00 52 4c 00 00 00 52 13 00 00 00 28 00 00 00 00 28 00 00 00 00 28 00 00 00 00 73 52 00 00 ...RL...R....(....(....(....sR..
22c0 00 2f 62 75 69 6c 64 62 6f 74 2f 74 6d 70 2f 62 75 69 6c 64 68 6f 73 74 2f 69 6e 73 74 61 6c 6c ./buildbot/tmp/buildhost/install
22e0 2f 70 72 65 62 75 69 6c 74 2f 6c 69 6e 75 78 2d 78 38 36 5f 36 34 2f 6c 69 62 2f 70 79 74 68 6f /prebuilt/linux-x86_64/lib/pytho
2300 6e 32 2e 37 2f 72 6f 62 6f 74 70 61 72 73 65 72 2e 70 79 74 08 00 00 00 3c 6d 6f 64 75 6c 65 3e n2.7/robotparser.pyt....<module>
2320 0b 00 00 00 73 0e 00 00 00 06 01 0c 01 0c 02 09 03 13 8a 13 11 13 24 ....s.................$