From 554fd8c5195424bdbcabf5de30fdc183aba391bd Mon Sep 17 00:00:00 2001 From: upstream source tree Date: Sun, 15 Mar 2015 20:14:05 -0400 Subject: obtained gcc-4.6.4.tar.bz2 from upstream website; verified gcc-4.6.4.tar.bz2.sig; imported gcc-4.6.4 source tree from verified upstream tarball. downloading a git-generated archive based on the 'upstream' tag should provide you with a source tree that is binary identical to the one extracted from the above tarball. if you have obtained the source via the command 'git clone', however, do note that line-endings of files in your working directory might differ from line-endings of the respective files in the upstream repository. --- libgo/go/archive/tar/common.go | 75 + libgo/go/archive/tar/reader.go | 226 ++ libgo/go/archive/tar/reader_test.go | 274 ++ libgo/go/archive/tar/testdata/gnu.tar | Bin 0 -> 3072 bytes libgo/go/archive/tar/testdata/small.txt | 1 + libgo/go/archive/tar/testdata/small2.txt | 1 + libgo/go/archive/tar/testdata/star.tar | Bin 0 -> 3072 bytes libgo/go/archive/tar/testdata/v7.tar | Bin 0 -> 3584 bytes libgo/go/archive/tar/testdata/writer-big.tar | Bin 0 -> 4096 bytes libgo/go/archive/tar/testdata/writer.tar | Bin 0 -> 3072 bytes libgo/go/archive/tar/writer.go | 205 + libgo/go/archive/tar/writer_test.go | 154 + libgo/go/archive/zip/reader.go | 278 ++ libgo/go/archive/zip/reader_test.go | 180 + libgo/go/archive/zip/struct.go | 33 + libgo/go/archive/zip/testdata/gophercolor16x16.png | Bin 0 -> 785 bytes libgo/go/archive/zip/testdata/r.zip | Bin 0 -> 440 bytes libgo/go/archive/zip/testdata/readme.notzip | Bin 0 -> 1905 bytes libgo/go/archive/zip/testdata/readme.zip | Bin 0 -> 1885 bytes libgo/go/archive/zip/testdata/test.zip | Bin 0 -> 1170 bytes libgo/go/asn1/asn1.go | 785 ++++ libgo/go/asn1/asn1_test.go | 634 +++ libgo/go/asn1/common.go | 149 + libgo/go/asn1/marshal.go | 482 +++ libgo/go/asn1/marshal_test.go | 100 + libgo/go/big/arith.go | 259 ++ libgo/go/big/arith_decl.go | 18 + libgo/go/big/arith_test.go | 342 ++ libgo/go/big/calibrate_test.go | 92 + libgo/go/big/hilbert_test.go | 173 + libgo/go/big/int.go | 741 ++++ libgo/go/big/int_test.go | 1055 +++++ libgo/go/big/nat.go | 1067 +++++ libgo/go/big/nat_test.go | 358 ++ libgo/go/big/rat.go | 326 ++ libgo/go/big/rat_test.go | 282 ++ libgo/go/bufio/bufio.go | 526 +++ libgo/go/bufio/bufio_test.go | 572 +++ libgo/go/bytes/buffer.go | 315 ++ libgo/go/bytes/buffer_test.go | 349 ++ libgo/go/bytes/bytes.go | 602 +++ libgo/go/bytes/bytes_decl.go | 8 + libgo/go/bytes/bytes_test.go | 844 ++++ libgo/go/bytes/export_test.go | 8 + libgo/go/bytes/indexbyte.c | 28 + libgo/go/cmath/abs.go | 12 + libgo/go/cmath/asin.go | 170 + libgo/go/cmath/cmath_test.go | 853 ++++ libgo/go/cmath/conj.go | 8 + libgo/go/cmath/exp.go | 55 + libgo/go/cmath/isinf.go | 21 + libgo/go/cmath/isnan.go | 25 + libgo/go/cmath/log.go | 64 + libgo/go/cmath/phase.go | 11 + libgo/go/cmath/polar.go | 12 + libgo/go/cmath/pow.go | 60 + libgo/go/cmath/rect.go | 13 + libgo/go/cmath/sin.go | 132 + libgo/go/cmath/sqrt.go | 103 + libgo/go/cmath/tan.go | 184 + libgo/go/compress/flate/deflate.go | 521 +++ libgo/go/compress/flate/deflate_test.go | 389 ++ libgo/go/compress/flate/flate_test.go | 139 + libgo/go/compress/flate/huffman_bit_writer.go | 506 +++ libgo/go/compress/flate/huffman_code.go | 373 ++ libgo/go/compress/flate/inflate.go | 620 +++ libgo/go/compress/flate/reverse_bits.go | 48 + libgo/go/compress/flate/token.go | 103 + libgo/go/compress/flate/util.go | 72 + libgo/go/compress/gzip/gunzip.go | 230 ++ libgo/go/compress/gzip/gunzip_test.go | 305 ++ libgo/go/compress/gzip/gzip.go | 187 + libgo/go/compress/gzip/gzip_test.go | 84 + libgo/go/compress/zlib/reader.go | 112 + libgo/go/compress/zlib/reader_test.go | 95 + libgo/go/compress/zlib/testdata/e.txt | 1 + libgo/go/compress/zlib/testdata/pi.txt | 1 + libgo/go/compress/zlib/writer.go | 106 + libgo/go/compress/zlib/writer_test.go | 106 + libgo/go/container/heap/heap.go | 102 + libgo/go/container/heap/heap_test.go | 166 + libgo/go/container/list/list.go | 211 + libgo/go/container/list/list_test.go | 209 + libgo/go/container/ring/ring.go | 153 + libgo/go/container/ring/ring_test.go | 240 ++ libgo/go/container/vector/defs.go | 51 + libgo/go/container/vector/intvector.go | 208 + libgo/go/container/vector/intvector_test.go | 344 ++ libgo/go/container/vector/nogen_test.go | 76 + libgo/go/container/vector/numbers_test.go | 122 + libgo/go/container/vector/stringvector.go | 208 + libgo/go/container/vector/stringvector_test.go | 344 ++ libgo/go/container/vector/vector.go | 208 + libgo/go/container/vector/vector_test.go | 344 ++ libgo/go/crypto/aes/aes_test.go | 350 ++ libgo/go/crypto/aes/block.go | 176 + libgo/go/crypto/aes/cipher.go | 71 + libgo/go/crypto/aes/const.go | 362 ++ libgo/go/crypto/block/cbc.go | 71 + libgo/go/crypto/block/cfb.go | 96 + libgo/go/crypto/block/cfb_aes_test.go | 311 ++ libgo/go/crypto/block/cipher.go | 57 + libgo/go/crypto/block/cmac.go | 105 + libgo/go/crypto/block/cmac_aes_test.go | 130 + libgo/go/crypto/block/ctr.go | 67 + libgo/go/crypto/block/eax.go | 253 ++ libgo/go/crypto/block/eax_aes_test.go | 140 + libgo/go/crypto/block/ecb.go | 270 ++ libgo/go/crypto/block/ecb_aes_test.go | 127 + libgo/go/crypto/block/ecb_test.go | 181 + libgo/go/crypto/block/ofb.go | 60 + libgo/go/crypto/block/ofb_aes_test.go | 108 + libgo/go/crypto/block/xor.go | 124 + libgo/go/crypto/block/xor_test.go | 168 + libgo/go/crypto/blowfish/block.go | 101 + libgo/go/crypto/blowfish/blowfish_test.go | 192 + libgo/go/crypto/blowfish/cipher.go | 79 + libgo/go/crypto/blowfish/const.go | 199 + libgo/go/crypto/cast5/cast5.go | 536 +++ libgo/go/crypto/cast5/cast5_test.go | 104 + libgo/go/crypto/cipher/cbc.go | 78 + libgo/go/crypto/cipher/cbc_aes_test.go | 89 + libgo/go/crypto/cipher/cfb.go | 64 + libgo/go/crypto/cipher/cfb_test.go | 35 + libgo/go/crypto/cipher/cipher.go | 63 + libgo/go/crypto/cipher/common_test.go | 28 + libgo/go/crypto/cipher/ctr.go | 51 + libgo/go/crypto/cipher/ctr_aes_test.go | 101 + libgo/go/crypto/cipher/io.go | 57 + libgo/go/crypto/cipher/ocfb.go | 112 + libgo/go/crypto/cipher/ocfb_test.go | 39 + libgo/go/crypto/cipher/ofb.go | 44 + libgo/go/crypto/cipher/ofb_test.go | 101 + libgo/go/crypto/elliptic/elliptic.go | 376 ++ libgo/go/crypto/elliptic/elliptic_test.go | 331 ++ libgo/go/crypto/hmac/hmac.go | 100 + libgo/go/crypto/hmac/hmac_test.go | 205 + libgo/go/crypto/md4/md4.go | 112 + libgo/go/crypto/md4/md4_test.go | 71 + libgo/go/crypto/md4/md4block.go | 89 + libgo/go/crypto/md5/md5.go | 112 + libgo/go/crypto/md5/md5_test.go | 71 + libgo/go/crypto/md5/md5block.go | 172 + libgo/go/crypto/ocsp/ocsp.go | 203 + libgo/go/crypto/ocsp/ocsp_test.go | 97 + libgo/go/crypto/openpgp/armor/armor.go | 220 + libgo/go/crypto/openpgp/armor/armor_test.go | 97 + libgo/go/crypto/openpgp/armor/encode.go | 162 + libgo/go/crypto/openpgp/error/error.go | 46 + libgo/go/crypto/openpgp/s2k/s2k.go | 146 + libgo/go/crypto/openpgp/s2k/s2k_test.go | 94 + libgo/go/crypto/rand/rand.go | 21 + libgo/go/crypto/rand/rand_test.go | 27 + libgo/go/crypto/rand/rand_unix.go | 125 + libgo/go/crypto/rand/rand_windows.go | 43 + libgo/go/crypto/rc4/rc4.go | 66 + libgo/go/crypto/rc4/rc4_test.go | 59 + libgo/go/crypto/ripemd160/ripemd160.go | 113 + libgo/go/crypto/ripemd160/ripemd160_test.go | 64 + libgo/go/crypto/ripemd160/ripemd160block.go | 161 + libgo/go/crypto/rsa/pkcs1v15.go | 273 ++ libgo/go/crypto/rsa/pkcs1v15_test.go | 220 + libgo/go/crypto/rsa/rsa.go | 445 ++ libgo/go/crypto/rsa/rsa_test.go | 250 ++ libgo/go/crypto/sha1/sha1.go | 114 + libgo/go/crypto/sha1/sha1_test.go | 73 + libgo/go/crypto/sha1/sha1block.go | 81 + libgo/go/crypto/sha256/sha256.go | 159 + libgo/go/crypto/sha256/sha256_test.go | 125 + libgo/go/crypto/sha256/sha256block.go | 129 + libgo/go/crypto/sha512/sha512.go | 163 + libgo/go/crypto/sha512/sha512_test.go | 125 + libgo/go/crypto/sha512/sha512block.go | 144 + libgo/go/crypto/subtle/constant_time.go | 57 + libgo/go/crypto/subtle/constant_time_test.go | 105 + libgo/go/crypto/tls/alert.go | 73 + libgo/go/crypto/tls/ca_set.go | 89 + libgo/go/crypto/tls/cipher_suites.go | 102 + libgo/go/crypto/tls/common.go | 258 ++ libgo/go/crypto/tls/conn.go | 801 ++++ libgo/go/crypto/tls/conn_test.go | 52 + libgo/go/crypto/tls/generate_cert.go | 70 + libgo/go/crypto/tls/handshake_client.go | 300 ++ libgo/go/crypto/tls/handshake_client_test.go | 211 + libgo/go/crypto/tls/handshake_messages.go | 904 +++++ libgo/go/crypto/tls/handshake_messages_test.go | 202 + libgo/go/crypto/tls/handshake_server.go | 285 ++ libgo/go/crypto/tls/handshake_server_test.go | 516 +++ libgo/go/crypto/tls/key_agreement.go | 246 ++ libgo/go/crypto/tls/prf.go | 153 + libgo/go/crypto/tls/prf_test.go | 104 + libgo/go/crypto/tls/tls.go | 167 + libgo/go/crypto/twofish/twofish.go | 358 ++ libgo/go/crypto/twofish/twofish_test.go | 129 + libgo/go/crypto/x509/x509.go | 854 ++++ libgo/go/crypto/x509/x509_test.go | 198 + libgo/go/crypto/xtea/block.go | 66 + libgo/go/crypto/xtea/cipher.go | 92 + libgo/go/crypto/xtea/xtea_test.go | 246 ++ libgo/go/debug/dwarf/buf.go | 154 + libgo/go/debug/dwarf/const.go | 433 ++ libgo/go/debug/dwarf/entry.go | 343 ++ libgo/go/debug/dwarf/open.go | 80 + libgo/go/debug/dwarf/testdata/typedef.c | 79 + libgo/go/debug/dwarf/testdata/typedef.elf | Bin 0 -> 10837 bytes libgo/go/debug/dwarf/testdata/typedef.macho | Bin 0 -> 5256 bytes libgo/go/debug/dwarf/type.go | 583 +++ libgo/go/debug/dwarf/type_test.go | 112 + libgo/go/debug/dwarf/unit.go | 62 + libgo/go/debug/elf/elf.go | 1506 +++++++ libgo/go/debug/elf/elf_test.go | 49 + libgo/go/debug/elf/file.go | 605 +++ libgo/go/debug/elf/file_test.go | 180 + libgo/go/debug/elf/testdata/gcc-386-freebsd-exec | Bin 0 -> 5742 bytes libgo/go/debug/elf/testdata/gcc-amd64-linux-exec | Bin 0 -> 8844 bytes .../testdata/go-relocation-test-gcc424-x86-64.o | Bin 0 -> 3088 bytes .../testdata/go-relocation-test-gcc441-x86-64.o | Bin 0 -> 2936 bytes .../elf/testdata/go-relocation-test-gcc441-x86.o | Bin 0 -> 1884 bytes libgo/go/debug/gosym/pclinetest.h | 7 + libgo/go/debug/gosym/pclinetest.s | 89 + libgo/go/debug/gosym/pclntab.go | 82 + libgo/go/debug/gosym/pclntab_test.go | 207 + libgo/go/debug/gosym/symtab.go | 548 +++ libgo/go/debug/macho/file.go | 517 +++ libgo/go/debug/macho/file_test.go | 167 + libgo/go/debug/macho/macho.go | 305 ++ libgo/go/debug/macho/testdata/gcc-386-darwin-exec | Bin 0 -> 12588 bytes .../go/debug/macho/testdata/gcc-amd64-darwin-exec | Bin 0 -> 8512 bytes .../macho/testdata/gcc-amd64-darwin-exec-debug | Bin 0 -> 4540 bytes libgo/go/debug/macho/testdata/hello.c | 8 + libgo/go/debug/pe/file.go | 309 ++ libgo/go/debug/pe/file_test.go | 99 + libgo/go/debug/pe/pe.go | 51 + libgo/go/debug/pe/testdata/gcc-386-mingw-exec | Bin 0 -> 29941 bytes libgo/go/debug/pe/testdata/gcc-386-mingw-obj | Bin 0 -> 2372 bytes libgo/go/debug/pe/testdata/hello.c | 8 + libgo/go/debug/proc/proc.go | 222 + libgo/go/debug/proc/proc_darwin.go | 17 + libgo/go/debug/proc/proc_freebsd.go | 17 + libgo/go/debug/proc/proc_linux.go | 1316 ++++++ libgo/go/debug/proc/proc_rtems.go | 17 + libgo/go/debug/proc/proc_solaris.go | 17 + libgo/go/debug/proc/proc_windows.go | 17 + libgo/go/debug/proc/ptrace-nptl.txt | 132 + libgo/go/debug/proc/regs_darwin_386.go | 5 + libgo/go/debug/proc/regs_darwin_amd64.go | 5 + libgo/go/debug/proc/regs_freebsd_386.go | 5 + libgo/go/debug/proc/regs_freebsd_amd64.go | 5 + libgo/go/debug/proc/regs_linux_386.go | 143 + libgo/go/debug/proc/regs_linux_amd64.go | 191 + libgo/go/debug/proc/regs_linux_arm.go | 39 + libgo/go/debug/proc/regs_windows_386.go | 5 + libgo/go/debug/proc/regs_windows_amd64.go | 5 + libgo/go/ebnf/ebnf.go | 248 ++ libgo/go/ebnf/ebnf_test.go | 77 + libgo/go/ebnf/parser.go | 208 + libgo/go/encoding/ascii85/ascii85.go | 300 ++ libgo/go/encoding/ascii85/ascii85_test.go | 188 + libgo/go/encoding/base32/base32.go | 368 ++ libgo/go/encoding/base32/base32_test.go | 194 + libgo/go/encoding/base64/base64.go | 329 ++ libgo/go/encoding/base64/base64_test.go | 196 + libgo/go/encoding/binary/binary.go | 409 ++ libgo/go/encoding/binary/binary_test.go | 162 + libgo/go/encoding/git85/git.go | 277 ++ libgo/go/encoding/git85/git_test.go | 194 + libgo/go/encoding/hex/hex.go | 101 + libgo/go/encoding/hex/hex_test.go | 149 + libgo/go/encoding/line/line.go | 95 + libgo/go/encoding/line/line_test.go | 89 + libgo/go/encoding/pem/pem.go | 257 ++ libgo/go/encoding/pem/pem_test.go | 390 ++ libgo/go/exec/exec.go | 183 + libgo/go/exec/exec_test.go | 120 + libgo/go/exec/lp_unix.go | 45 + libgo/go/exec/lp_windows.go | 66 + libgo/go/exp/README | 3 + libgo/go/exp/datafmt/datafmt.go | 731 ++++ libgo/go/exp/datafmt/datafmt_test.go | 351 ++ libgo/go/exp/datafmt/parser.go | 386 ++ libgo/go/exp/draw/draw.go | 363 ++ libgo/go/exp/draw/draw_test.go | 228 ++ libgo/go/exp/draw/event.go | 56 + libgo/go/exp/draw/x11/auth.go | 93 + libgo/go/exp/draw/x11/conn.go | 622 +++ libgo/go/exp/eval/abort.go | 85 + libgo/go/exp/eval/bridge.go | 169 + libgo/go/exp/eval/compiler.go | 92 + libgo/go/exp/eval/eval_test.go | 259 ++ libgo/go/exp/eval/expr.go | 2015 ++++++++++ libgo/go/exp/eval/expr1.go | 1874 +++++++++ libgo/go/exp/eval/expr_test.go | 355 ++ libgo/go/exp/eval/func.go | 70 + libgo/go/exp/eval/scope.go | 207 + libgo/go/exp/eval/stmt.go | 1302 ++++++ libgo/go/exp/eval/stmt_test.go | 343 ++ libgo/go/exp/eval/type.go | 1252 ++++++ libgo/go/exp/eval/typec.go | 409 ++ libgo/go/exp/eval/value.go | 586 +++ libgo/go/exp/eval/world.go | 188 + libgo/go/exp/ogle/abort.go | 35 + libgo/go/exp/ogle/arch.go | 125 + libgo/go/exp/ogle/cmd.go | 373 ++ libgo/go/exp/ogle/event.go | 280 ++ libgo/go/exp/ogle/frame.go | 212 + libgo/go/exp/ogle/goroutine.go | 117 + libgo/go/exp/ogle/main.go | 9 + libgo/go/exp/ogle/process.go | 521 +++ libgo/go/exp/ogle/rruntime.go | 271 ++ libgo/go/exp/ogle/rtype.go | 288 ++ libgo/go/exp/ogle/rvalue.go | 515 +++ libgo/go/exp/ogle/vars.go | 272 ++ libgo/go/expvar/expvar.go | 299 ++ libgo/go/expvar/expvar_test.go | 154 + libgo/go/flag/export_test.go | 32 + libgo/go/flag/flag.go | 480 +++ libgo/go/flag/flag_test.go | 194 + libgo/go/fmt/doc.go | 164 + libgo/go/fmt/fmt_test.go | 650 +++ libgo/go/fmt/format.go | 398 ++ libgo/go/fmt/print.go | 921 +++++ libgo/go/fmt/scan.go | 985 +++++ libgo/go/fmt/scan_test.go | 659 +++ libgo/go/fmt/stringer_test.go | 61 + libgo/go/go/ast/ast.go | 959 +++++ libgo/go/go/ast/filter.go | 429 ++ libgo/go/go/ast/print.go | 217 + libgo/go/go/ast/scope.go | 242 ++ libgo/go/go/ast/walk.go | 396 ++ libgo/go/go/doc/comment.go | 357 ++ libgo/go/go/doc/doc.go | 650 +++ libgo/go/go/parser/interface.go | 208 + libgo/go/go/parser/parser.go | 1885 +++++++++ libgo/go/go/parser/parser_test.go | 112 + libgo/go/go/printer/nodes.go | 1493 +++++++ libgo/go/go/printer/printer.go | 1140 ++++++ libgo/go/go/printer/printer_test.go | 138 + libgo/go/go/printer/testdata/comments.golden | 483 +++ libgo/go/go/printer/testdata/comments.input | 483 +++ libgo/go/go/printer/testdata/comments.x | 57 + libgo/go/go/printer/testdata/declarations.golden | 715 ++++ libgo/go/go/printer/testdata/declarations.input | 703 ++++ libgo/go/go/printer/testdata/empty.golden | 5 + libgo/go/go/printer/testdata/empty.input | 5 + libgo/go/go/printer/testdata/expressions.golden | 554 +++ libgo/go/go/printer/testdata/expressions.input | 548 +++ libgo/go/go/printer/testdata/expressions.raw | 554 +++ libgo/go/go/printer/testdata/linebreaks.golden | 223 + libgo/go/go/printer/testdata/linebreaks.input | 223 + libgo/go/go/printer/testdata/statements.golden | 417 ++ libgo/go/go/printer/testdata/statements.input | 338 ++ libgo/go/go/scanner/errors.go | 186 + libgo/go/go/scanner/scanner.go | 714 ++++ libgo/go/go/scanner/scanner_test.go | 672 ++++ libgo/go/go/token/position.go | 409 ++ libgo/go/go/token/position_test.go | 158 + libgo/go/go/token/token.go | 320 ++ libgo/go/go/typechecker/scope.go | 119 + libgo/go/go/typechecker/testdata/test0.go | 94 + libgo/go/go/typechecker/testdata/test1.go | 13 + libgo/go/go/typechecker/testdata/test3.go | 38 + libgo/go/go/typechecker/testdata/test4.go | 11 + libgo/go/go/typechecker/typechecker.go | 484 +++ libgo/go/go/typechecker/typechecker_test.go | 168 + libgo/go/go/typechecker/universe.go | 38 + libgo/go/gob/codec_test.go | 1355 +++++++ libgo/go/gob/decode.go | 1020 +++++ libgo/go/gob/decoder.go | 164 + libgo/go/gob/doc.go | 307 ++ libgo/go/gob/encode.go | 573 +++ libgo/go/gob/encoder.go | 207 + libgo/go/gob/encoder_test.go | 385 ++ libgo/go/gob/error.go | 41 + libgo/go/gob/type.go | 539 +++ libgo/go/gob/type_test.go | 153 + libgo/go/hash/adler32/adler32.go | 88 + libgo/go/hash/adler32/adler32_test.go | 63 + libgo/go/hash/crc32/crc32.go | 111 + libgo/go/hash/crc32/crc32_test.go | 76 + libgo/go/hash/crc64/crc64.go | 96 + libgo/go/hash/crc64/crc64_test.go | 78 + libgo/go/hash/hash.go | 36 + libgo/go/html/doc.go | 106 + libgo/go/html/entity.go | 2250 +++++++++++ libgo/go/html/entity_test.go | 26 + libgo/go/html/escape.go | 224 ++ libgo/go/html/parse.go | 666 +++ libgo/go/html/parse_test.go | 158 + libgo/go/html/testdata/webkit/README | 28 + libgo/go/html/testdata/webkit/comments01.dat | 126 + libgo/go/html/testdata/webkit/doctype01.dat | 335 ++ libgo/go/html/testdata/webkit/dom2string.js | 135 + libgo/go/html/testdata/webkit/entities01.dat | 612 +++ libgo/go/html/testdata/webkit/entities02.dat | 129 + libgo/go/html/testdata/webkit/scriptdata01.dat | 308 ++ libgo/go/html/testdata/webkit/tests1.dat | 1949 +++++++++ libgo/go/html/testdata/webkit/tests10.dat | 430 ++ libgo/go/html/testdata/webkit/tests11.dat | 482 +++ libgo/go/html/testdata/webkit/tests12.dat | 62 + libgo/go/html/testdata/webkit/tests13.dat | 9 + libgo/go/html/testdata/webkit/tests14.dat | 74 + libgo/go/html/testdata/webkit/tests15.dat | 208 + libgo/go/html/testdata/webkit/tests16.dat | 2277 +++++++++++ libgo/go/html/testdata/webkit/tests2.dat | 738 ++++ libgo/go/html/testdata/webkit/tests3.dat | 293 ++ libgo/go/html/testdata/webkit/tests4.dat | 59 + libgo/go/html/testdata/webkit/tests5.dat | 191 + libgo/go/html/testdata/webkit/tests6.dat | 653 +++ libgo/go/html/testdata/webkit/tests7.dat | 390 ++ libgo/go/html/testdata/webkit/tests8.dat | 148 + libgo/go/html/testdata/webkit/tests9.dat | 430 ++ libgo/go/html/testdata/webkit/webkit01.dat | 211 + libgo/go/html/token.go | 398 ++ libgo/go/html/token_test.go | 231 ++ libgo/go/http/chunked.go | 56 + libgo/go/http/client.go | 236 ++ libgo/go/http/client_test.go | 40 + libgo/go/http/dump.go | 76 + libgo/go/http/fs.go | 265 ++ libgo/go/http/fs_test.go | 172 + libgo/go/http/lex.go | 144 + libgo/go/http/lex_test.go | 70 + libgo/go/http/persist.go | 303 ++ libgo/go/http/pprof/pprof.go | 92 + libgo/go/http/readrequest_test.go | 132 + libgo/go/http/request.go | 693 ++++ libgo/go/http/request_test.go | 155 + libgo/go/http/requestwrite_test.go | 139 + libgo/go/http/response.go | 251 ++ libgo/go/http/response_test.go | 203 + libgo/go/http/responsewrite_test.go | 85 + libgo/go/http/serve_test.go | 220 + libgo/go/http/server.go | 766 ++++ libgo/go/http/status.go | 106 + libgo/go/http/testdata/file | 1 + libgo/go/http/transfer.go | 441 ++ libgo/go/http/url.go | 595 +++ libgo/go/http/url_test.go | 675 ++++ libgo/go/image/color.go | 251 ++ libgo/go/image/format.go | 86 + libgo/go/image/geom.go | 223 + libgo/go/image/image.go | 506 +++ libgo/go/image/jpeg/huffman.go | 190 + libgo/go/image/jpeg/idct.go | 190 + libgo/go/image/jpeg/reader.go | 455 +++ libgo/go/image/names.go | 67 + libgo/go/image/png/reader.go | 588 +++ libgo/go/image/png/reader_test.go | 190 + libgo/go/image/png/testdata/pngsuite/README | 9 + .../go/image/png/testdata/pngsuite/README.original | 85 + libgo/go/image/png/testdata/pngsuite/basn0g01.png | Bin 0 -> 164 bytes libgo/go/image/png/testdata/pngsuite/basn0g01.sng | 41 + libgo/go/image/png/testdata/pngsuite/basn0g02.png | Bin 0 -> 104 bytes libgo/go/image/png/testdata/pngsuite/basn0g02.sng | 41 + libgo/go/image/png/testdata/pngsuite/basn0g04.png | Bin 0 -> 145 bytes libgo/go/image/png/testdata/pngsuite/basn0g04.sng | 41 + libgo/go/image/png/testdata/pngsuite/basn0g08.png | Bin 0 -> 138 bytes libgo/go/image/png/testdata/pngsuite/basn0g08.sng | 41 + libgo/go/image/png/testdata/pngsuite/basn0g16.png | Bin 0 -> 167 bytes libgo/go/image/png/testdata/pngsuite/basn0g16.sng | 41 + libgo/go/image/png/testdata/pngsuite/basn2c08.png | Bin 0 -> 145 bytes libgo/go/image/png/testdata/pngsuite/basn2c08.sng | 41 + libgo/go/image/png/testdata/pngsuite/basn2c16.png | Bin 0 -> 302 bytes libgo/go/image/png/testdata/pngsuite/basn2c16.sng | 41 + libgo/go/image/png/testdata/pngsuite/basn3p01.png | Bin 0 -> 112 bytes libgo/go/image/png/testdata/pngsuite/basn3p01.sng | 45 + libgo/go/image/png/testdata/pngsuite/basn3p02.png | Bin 0 -> 146 bytes libgo/go/image/png/testdata/pngsuite/basn3p02.sng | 50 + libgo/go/image/png/testdata/pngsuite/basn3p04.png | Bin 0 -> 216 bytes libgo/go/image/png/testdata/pngsuite/basn3p04.sng | 61 + libgo/go/image/png/testdata/pngsuite/basn3p08.png | Bin 0 -> 1286 bytes libgo/go/image/png/testdata/pngsuite/basn3p08.sng | 299 ++ libgo/go/image/png/testdata/pngsuite/basn4a08.png | Bin 0 -> 126 bytes libgo/go/image/png/testdata/pngsuite/basn4a08.sng | 41 + libgo/go/image/png/testdata/pngsuite/basn4a16.png | Bin 0 -> 2206 bytes libgo/go/image/png/testdata/pngsuite/basn4a16.sng | 41 + libgo/go/image/png/testdata/pngsuite/basn6a08.png | Bin 0 -> 184 bytes libgo/go/image/png/testdata/pngsuite/basn6a08.sng | 41 + libgo/go/image/png/testdata/pngsuite/basn6a16.png | Bin 0 -> 3435 bytes libgo/go/image/png/testdata/pngsuite/basn6a16.sng | 41 + libgo/go/image/png/writer.go | 437 ++ libgo/go/image/png/writer_test.go | 86 + libgo/go/index/suffixarray/qsufsort.go | 164 + libgo/go/index/suffixarray/suffixarray.go | 182 + libgo/go/index/suffixarray/suffixarray_test.go | 234 ++ libgo/go/io/io.go | 369 ++ libgo/go/io/io_test.go | 156 + libgo/go/io/ioutil/ioutil.go | 90 + libgo/go/io/ioutil/ioutil_test.go | 92 + libgo/go/io/ioutil/tempfile.go | 60 + libgo/go/io/ioutil/tempfile_test.go | 33 + libgo/go/io/multi.go | 60 + libgo/go/io/multi_test.go | 88 + libgo/go/io/pipe.go | 305 ++ libgo/go/io/pipe_test.go | 271 ++ libgo/go/json/decode.go | 861 ++++ libgo/go/json/decode_test.go | 517 +++ libgo/go/json/encode.go | 332 ++ libgo/go/json/indent.go | 116 + libgo/go/json/scanner.go | 618 +++ libgo/go/json/scanner_test.go | 260 ++ libgo/go/json/stream.go | 184 + libgo/go/json/stream_test.go | 122 + libgo/go/log/log.go | 278 ++ libgo/go/log/log_test.go | 86 + libgo/go/math/acosh.go | 62 + libgo/go/math/all_test.go | 2737 +++++++++++++ libgo/go/math/asin.go | 50 + libgo/go/math/asinh.go | 72 + libgo/go/math/atan.go | 62 + libgo/go/math/atan2.go | 71 + libgo/go/math/atanh.go | 79 + libgo/go/math/bits.go | 59 + libgo/go/math/cbrt.go | 79 + libgo/go/math/const.go | 53 + libgo/go/math/copysign.go | 12 + libgo/go/math/erf.go | 340 ++ libgo/go/math/exp.go | 14 + libgo/go/math/exp2.go | 10 + libgo/go/math/exp_port.go | 192 + libgo/go/math/exp_test.go | 10 + libgo/go/math/expm1.go | 238 ++ libgo/go/math/fabs.go | 21 + libgo/go/math/fdim.go | 29 + libgo/go/math/floor.go | 53 + libgo/go/math/fmod.go | 48 + libgo/go/math/frexp.go | 33 + libgo/go/math/gamma.go | 188 + libgo/go/math/hypot.go | 41 + libgo/go/math/hypot_port.go | 63 + libgo/go/math/hypot_test.go | 9 + libgo/go/math/j0.go | 433 ++ libgo/go/math/j1.go | 426 ++ libgo/go/math/jn.go | 310 ++ libgo/go/math/ldexp.go | 45 + libgo/go/math/lgamma.go | 350 ++ libgo/go/math/log.go | 123 + libgo/go/math/log10.go | 13 + libgo/go/math/log10_decl.go | 8 + libgo/go/math/log1p.go | 200 + libgo/go/math/logb.go | 54 + libgo/go/math/modf.go | 33 + libgo/go/math/nextafter.go | 29 + libgo/go/math/pow.go | 139 + libgo/go/math/pow10.go | 30 + libgo/go/math/remainder.go | 85 + libgo/go/math/signbit.go | 10 + libgo/go/math/sin.go | 66 + libgo/go/math/sincos.go | 13 + libgo/go/math/sinh.go | 68 + libgo/go/math/sqrt.go | 28 + libgo/go/math/sqrt_decl.go | 7 + libgo/go/math/sqrt_port.go | 143 + libgo/go/math/sqrt_test.go | 9 + libgo/go/math/tan.go | 65 + libgo/go/math/tanh.go | 28 + libgo/go/math/unsafe.go | 21 + libgo/go/mime/grammar.go | 36 + libgo/go/mime/mediatype.go | 120 + libgo/go/mime/mediatype_test.go | 117 + libgo/go/mime/mime_test.go | 27 + libgo/go/mime/multipart/multipart.go | 280 ++ libgo/go/mime/multipart/multipart_test.go | 204 + libgo/go/mime/test.types | 8 + libgo/go/mime/type.go | 104 + libgo/go/net/dial.go | 179 + libgo/go/net/dialgoogle_test.go | 107 + libgo/go/net/dict/dict.go | 212 + libgo/go/net/dnsclient.go | 417 ++ libgo/go/net/dnsconfig.go | 120 + libgo/go/net/dnsmsg.go | 743 ++++ libgo/go/net/dnsname_test.go | 69 + libgo/go/net/fd.go | 612 +++ libgo/go/net/fd_linux.go | 149 + libgo/go/net/fd_rtems.go | 137 + libgo/go/net/fd_windows.go | 555 +++ libgo/go/net/hosts.go | 86 + libgo/go/net/hosts_test.go | 54 + libgo/go/net/hosts_testdata | 12 + libgo/go/net/ip.go | 446 ++ libgo/go/net/ip_test.go | 94 + libgo/go/net/ipraw_test.go | 117 + libgo/go/net/iprawsock.go | 358 ++ libgo/go/net/ipsock.go | 236 ++ libgo/go/net/net.go | 192 + libgo/go/net/net_test.go | 126 + libgo/go/net/newpollserver.go | 41 + libgo/go/net/newpollserver_rtems.go | 78 + libgo/go/net/parse.go | 214 + libgo/go/net/parse_test.go | 50 + libgo/go/net/pipe.go | 62 + libgo/go/net/pipe_test.go | 57 + libgo/go/net/port.go | 70 + libgo/go/net/port_test.go | 53 + libgo/go/net/resolv_windows.go | 112 + libgo/go/net/server_test.go | 203 + libgo/go/net/sock.go | 181 + libgo/go/net/srv_test.go | 22 + libgo/go/net/tcpsock.go | 293 ++ libgo/go/net/textproto/pipeline.go | 117 + libgo/go/net/textproto/reader.go | 492 +++ libgo/go/net/textproto/reader_test.go | 140 + libgo/go/net/textproto/textproto.go | 122 + libgo/go/net/textproto/writer.go | 119 + libgo/go/net/textproto/writer_test.go | 35 + libgo/go/net/timeout_test.go | 57 + libgo/go/net/udpsock.go | 281 ++ libgo/go/net/unixsock.go | 449 +++ libgo/go/netchan/common.go | 325 ++ libgo/go/netchan/export.go | 390 ++ libgo/go/netchan/import.go | 243 ++ libgo/go/netchan/netchan_test.go | 515 +++ libgo/go/os/dir.go | 72 + libgo/go/os/dir_largefile.go | 12 + libgo/go/os/dir_regfile.go | 12 + libgo/go/os/env.go | 73 + libgo/go/os/env_test.go | 59 + libgo/go/os/env_unix.go | 96 + libgo/go/os/env_windows.go | 127 + libgo/go/os/error.go | 113 + libgo/go/os/exec.go | 154 + libgo/go/os/file.go | 438 ++ libgo/go/os/file_unix.go | 110 + libgo/go/os/getwd.go | 92 + libgo/go/os/inotify/inotify_linux.go | 291 ++ libgo/go/os/inotify/inotify_linux_test.go | 99 + libgo/go/os/os_test.go | 882 ++++ libgo/go/os/path.go | 116 + libgo/go/os/path_test.go | 181 + libgo/go/os/proc.go | 35 + libgo/go/os/signal/mkunix.sh | 24 + libgo/go/os/signal/signal.go | 48 + libgo/go/os/signal/signal_test.go | 19 + libgo/go/os/stat.go | 40 + libgo/go/os/sys_bsd.go | 19 + libgo/go/os/sys_linux.go | 28 + libgo/go/os/sys_uname.go | 25 + libgo/go/os/time.go | 20 + libgo/go/os/types.go | 56 + libgo/go/patch/apply.go | 54 + libgo/go/patch/git.go | 121 + libgo/go/patch/patch.go | 322 ++ libgo/go/patch/patch_test.go | 390 ++ libgo/go/patch/textdiff.go | 171 + libgo/go/path/match.go | 278 ++ libgo/go/path/match_test.go | 106 + libgo/go/path/path.go | 212 + libgo/go/path/path_test.go | 345 ++ libgo/go/path/path_unix.go | 11 + libgo/go/path/path_windows.go | 11 + libgo/go/rand/exp.go | 223 + libgo/go/rand/normal.go | 158 + libgo/go/rand/rand.go | 179 + libgo/go/rand/rand_test.go | 350 ++ libgo/go/rand/rng.go | 246 ++ libgo/go/rand/zipf.go | 73 + libgo/go/reflect/all_test.go | 1392 +++++++ libgo/go/reflect/deepequal.go | 135 + libgo/go/reflect/tostring_test.go | 96 + libgo/go/reflect/type.go | 743 ++++ libgo/go/reflect/value.go | 1242 ++++++ libgo/go/regexp/all_test.go | 426 ++ libgo/go/regexp/find_test.go | 459 +++ libgo/go/regexp/regexp.go | 1337 ++++++ libgo/go/rpc/client.go | 250 ++ libgo/go/rpc/debug.go | 90 + libgo/go/rpc/jsonrpc/all_test.go | 156 + libgo/go/rpc/jsonrpc/client.go | 121 + libgo/go/rpc/jsonrpc/server.go | 133 + libgo/go/rpc/server.go | 530 +++ libgo/go/rpc/server_test.go | 384 ++ libgo/go/runtime/chan_defs.go | 56 + libgo/go/runtime/debug.go | 159 + libgo/go/runtime/debug/stack.go | 90 + libgo/go/runtime/debug/stack_test.go | 55 + libgo/go/runtime/error.go | 133 + libgo/go/runtime/export_test.go | 17 + libgo/go/runtime/extern.go | 163 + libgo/go/runtime/hashmap_defs.go | 51 + libgo/go/runtime/iface_defs.go | 18 + libgo/go/runtime/malloc_defs.go | 130 + libgo/go/runtime/mheapmap32_defs.go | 23 + libgo/go/runtime/mheapmap64_defs.go | 31 + libgo/go/runtime/pprof/pprof.go | 108 + libgo/go/runtime/runtime_defs.go | 200 + libgo/go/runtime/sig.go | 16 + libgo/go/runtime/softfloat64.go | 498 +++ libgo/go/runtime/softfloat64_test.go | 198 + libgo/go/runtime/type.go | 206 + libgo/go/scanner/scanner.go | 644 +++ libgo/go/scanner/scanner_test.go | 482 +++ libgo/go/smtp/auth.go | 69 + libgo/go/smtp/smtp.go | 295 ++ libgo/go/smtp/smtp_test.go | 182 + libgo/go/sort/search.go | 110 + libgo/go/sort/search_test.go | 137 + libgo/go/sort/sort.go | 206 + libgo/go/sort/sort_test.go | 267 ++ libgo/go/strconv/atob.go | 28 + libgo/go/strconv/atob_test.go | 56 + libgo/go/strconv/atof.go | 413 ++ libgo/go/strconv/atof_test.go | 183 + libgo/go/strconv/atoi.go | 202 + libgo/go/strconv/atoi_test.go | 303 ++ libgo/go/strconv/decimal.go | 371 ++ libgo/go/strconv/decimal_test.go | 117 + libgo/go/strconv/fp_test.go | 149 + libgo/go/strconv/ftoa.go | 405 ++ libgo/go/strconv/ftoa_test.go | 145 + libgo/go/strconv/internal_test.go | 15 + libgo/go/strconv/itoa.go | 57 + libgo/go/strconv/itoa_test.go | 174 + libgo/go/strconv/quote.go | 264 ++ libgo/go/strconv/quote_test.go | 170 + libgo/go/strconv/testfp.txt | 181 + libgo/go/strings/reader.go | 61 + libgo/go/strings/strings.go | 559 +++ libgo/go/strings/strings_test.go | 776 ++++ libgo/go/sync/cas.c | 15 + libgo/go/sync/mutex.go | 61 + libgo/go/sync/mutex_test.go | 91 + libgo/go/sync/once.go | 35 + libgo/go/sync/once_test.go | 37 + libgo/go/sync/rwmutex.go | 75 + libgo/go/sync/rwmutex_test.go | 114 + libgo/go/sync/xadd_test.go | 9 + libgo/go/syslog/syslog.go | 150 + libgo/go/syslog/syslog_c.c | 19 + libgo/go/syslog/syslog_solaris.go | 37 + libgo/go/syslog/syslog_test.go | 95 + libgo/go/syslog/syslog_unix.go | 31 + libgo/go/tabwriter/tabwriter.go | 586 +++ libgo/go/tabwriter/tabwriter_test.go | 625 +++ libgo/go/template/format.go | 77 + libgo/go/template/template.go | 992 +++++ libgo/go/template/template_test.go | 660 +++ libgo/go/testing/benchmark.go | 195 + libgo/go/testing/iotest/logger.go | 55 + libgo/go/testing/iotest/reader.go | 69 + libgo/go/testing/iotest/writer.go | 38 + libgo/go/testing/quick/quick.go | 364 ++ libgo/go/testing/quick/quick_test.go | 147 + libgo/go/testing/script/script.go | 359 ++ libgo/go/testing/script/script_test.go | 75 + libgo/go/testing/testing.go | 174 + libgo/go/time/format.go | 618 +++ libgo/go/time/sleep.go | 151 + libgo/go/time/sleep_test.go | 134 + libgo/go/time/tick.go | 171 + libgo/go/time/tick_test.go | 45 + libgo/go/time/time.go | 229 ++ libgo/go/time/time_test.go | 341 ++ libgo/go/time/zoneinfo_unix.go | 267 ++ libgo/go/time/zoneinfo_windows.go | 192 + libgo/go/try/try.go | 174 + libgo/go/try/try_test.go | 60 + libgo/go/unicode/casetables.go | 21 + libgo/go/unicode/digit.go | 13 + libgo/go/unicode/digit_test.go | 126 + libgo/go/unicode/letter.go | 241 ++ libgo/go/unicode/letter_test.go | 377 ++ libgo/go/unicode/script_test.go | 247 ++ libgo/go/unicode/tables.go | 4238 ++++++++++++++++++++ libgo/go/utf16/utf16.go | 101 + libgo/go/utf16/utf16_test.go | 118 + libgo/go/utf8/string.go | 211 + libgo/go/utf8/string_test.go | 109 + libgo/go/utf8/utf8.go | 356 ++ libgo/go/utf8/utf8_test.go | 315 ++ libgo/go/websocket/client.go | 321 ++ libgo/go/websocket/server.go | 219 + libgo/go/websocket/websocket.go | 189 + libgo/go/websocket/websocket_test.go | 272 ++ libgo/go/xml/embed_test.go | 124 + libgo/go/xml/read.go | 620 +++ libgo/go/xml/read_test.go | 329 ++ libgo/go/xml/xml.go | 1617 ++++++++ libgo/go/xml/xml_test.go | 439 ++ 778 files changed, 182887 insertions(+) create mode 100644 libgo/go/archive/tar/common.go create mode 100644 libgo/go/archive/tar/reader.go create mode 100644 libgo/go/archive/tar/reader_test.go create mode 100644 libgo/go/archive/tar/testdata/gnu.tar create mode 100644 libgo/go/archive/tar/testdata/small.txt create mode 100644 libgo/go/archive/tar/testdata/small2.txt create mode 100644 libgo/go/archive/tar/testdata/star.tar create mode 100644 libgo/go/archive/tar/testdata/v7.tar create mode 100644 libgo/go/archive/tar/testdata/writer-big.tar create mode 100644 libgo/go/archive/tar/testdata/writer.tar create mode 100644 libgo/go/archive/tar/writer.go create mode 100644 libgo/go/archive/tar/writer_test.go create mode 100644 libgo/go/archive/zip/reader.go create mode 100644 libgo/go/archive/zip/reader_test.go create mode 100644 libgo/go/archive/zip/struct.go create mode 100644 libgo/go/archive/zip/testdata/gophercolor16x16.png create mode 100644 libgo/go/archive/zip/testdata/r.zip create mode 100644 libgo/go/archive/zip/testdata/readme.notzip create mode 100644 libgo/go/archive/zip/testdata/readme.zip create mode 100644 libgo/go/archive/zip/testdata/test.zip create mode 100644 libgo/go/asn1/asn1.go create mode 100644 libgo/go/asn1/asn1_test.go create mode 100644 libgo/go/asn1/common.go create mode 100644 libgo/go/asn1/marshal.go create mode 100644 libgo/go/asn1/marshal_test.go create mode 100644 libgo/go/big/arith.go create mode 100644 libgo/go/big/arith_decl.go create mode 100644 libgo/go/big/arith_test.go create mode 100644 libgo/go/big/calibrate_test.go create mode 100644 libgo/go/big/hilbert_test.go create mode 100644 libgo/go/big/int.go create mode 100644 libgo/go/big/int_test.go create mode 100644 libgo/go/big/nat.go create mode 100644 libgo/go/big/nat_test.go create mode 100644 libgo/go/big/rat.go create mode 100644 libgo/go/big/rat_test.go create mode 100644 libgo/go/bufio/bufio.go create mode 100644 libgo/go/bufio/bufio_test.go create mode 100644 libgo/go/bytes/buffer.go create mode 100644 libgo/go/bytes/buffer_test.go create mode 100644 libgo/go/bytes/bytes.go create mode 100644 libgo/go/bytes/bytes_decl.go create mode 100644 libgo/go/bytes/bytes_test.go create mode 100644 libgo/go/bytes/export_test.go create mode 100644 libgo/go/bytes/indexbyte.c create mode 100644 libgo/go/cmath/abs.go create mode 100644 libgo/go/cmath/asin.go create mode 100644 libgo/go/cmath/cmath_test.go create mode 100644 libgo/go/cmath/conj.go create mode 100644 libgo/go/cmath/exp.go create mode 100644 libgo/go/cmath/isinf.go create mode 100644 libgo/go/cmath/isnan.go create mode 100644 libgo/go/cmath/log.go create mode 100644 libgo/go/cmath/phase.go create mode 100644 libgo/go/cmath/polar.go create mode 100644 libgo/go/cmath/pow.go create mode 100644 libgo/go/cmath/rect.go create mode 100644 libgo/go/cmath/sin.go create mode 100644 libgo/go/cmath/sqrt.go create mode 100644 libgo/go/cmath/tan.go create mode 100644 libgo/go/compress/flate/deflate.go create mode 100644 libgo/go/compress/flate/deflate_test.go create mode 100644 libgo/go/compress/flate/flate_test.go create mode 100644 libgo/go/compress/flate/huffman_bit_writer.go create mode 100644 libgo/go/compress/flate/huffman_code.go create mode 100644 libgo/go/compress/flate/inflate.go create mode 100644 libgo/go/compress/flate/reverse_bits.go create mode 100644 libgo/go/compress/flate/token.go create mode 100644 libgo/go/compress/flate/util.go create mode 100644 libgo/go/compress/gzip/gunzip.go create mode 100644 libgo/go/compress/gzip/gunzip_test.go create mode 100644 libgo/go/compress/gzip/gzip.go create mode 100644 libgo/go/compress/gzip/gzip_test.go create mode 100644 libgo/go/compress/zlib/reader.go create mode 100644 libgo/go/compress/zlib/reader_test.go create mode 100644 libgo/go/compress/zlib/testdata/e.txt create mode 100644 libgo/go/compress/zlib/testdata/pi.txt create mode 100644 libgo/go/compress/zlib/writer.go create mode 100644 libgo/go/compress/zlib/writer_test.go create mode 100644 libgo/go/container/heap/heap.go create mode 100644 libgo/go/container/heap/heap_test.go create mode 100755 libgo/go/container/list/list.go create mode 100755 libgo/go/container/list/list_test.go create mode 100644 libgo/go/container/ring/ring.go create mode 100644 libgo/go/container/ring/ring_test.go create mode 100644 libgo/go/container/vector/defs.go create mode 100644 libgo/go/container/vector/intvector.go create mode 100644 libgo/go/container/vector/intvector_test.go create mode 100644 libgo/go/container/vector/nogen_test.go create mode 100644 libgo/go/container/vector/numbers_test.go create mode 100644 libgo/go/container/vector/stringvector.go create mode 100644 libgo/go/container/vector/stringvector_test.go create mode 100644 libgo/go/container/vector/vector.go create mode 100644 libgo/go/container/vector/vector_test.go create mode 100644 libgo/go/crypto/aes/aes_test.go create mode 100644 libgo/go/crypto/aes/block.go create mode 100644 libgo/go/crypto/aes/cipher.go create mode 100644 libgo/go/crypto/aes/const.go create mode 100644 libgo/go/crypto/block/cbc.go create mode 100644 libgo/go/crypto/block/cfb.go create mode 100644 libgo/go/crypto/block/cfb_aes_test.go create mode 100644 libgo/go/crypto/block/cipher.go create mode 100644 libgo/go/crypto/block/cmac.go create mode 100644 libgo/go/crypto/block/cmac_aes_test.go create mode 100644 libgo/go/crypto/block/ctr.go create mode 100644 libgo/go/crypto/block/eax.go create mode 100644 libgo/go/crypto/block/eax_aes_test.go create mode 100644 libgo/go/crypto/block/ecb.go create mode 100644 libgo/go/crypto/block/ecb_aes_test.go create mode 100644 libgo/go/crypto/block/ecb_test.go create mode 100644 libgo/go/crypto/block/ofb.go create mode 100644 libgo/go/crypto/block/ofb_aes_test.go create mode 100644 libgo/go/crypto/block/xor.go create mode 100644 libgo/go/crypto/block/xor_test.go create mode 100644 libgo/go/crypto/blowfish/block.go create mode 100644 libgo/go/crypto/blowfish/blowfish_test.go create mode 100644 libgo/go/crypto/blowfish/cipher.go create mode 100644 libgo/go/crypto/blowfish/const.go create mode 100644 libgo/go/crypto/cast5/cast5.go create mode 100644 libgo/go/crypto/cast5/cast5_test.go create mode 100644 libgo/go/crypto/cipher/cbc.go create mode 100644 libgo/go/crypto/cipher/cbc_aes_test.go create mode 100644 libgo/go/crypto/cipher/cfb.go create mode 100644 libgo/go/crypto/cipher/cfb_test.go create mode 100644 libgo/go/crypto/cipher/cipher.go create mode 100644 libgo/go/crypto/cipher/common_test.go create mode 100644 libgo/go/crypto/cipher/ctr.go create mode 100644 libgo/go/crypto/cipher/ctr_aes_test.go create mode 100644 libgo/go/crypto/cipher/io.go create mode 100644 libgo/go/crypto/cipher/ocfb.go create mode 100644 libgo/go/crypto/cipher/ocfb_test.go create mode 100644 libgo/go/crypto/cipher/ofb.go create mode 100644 libgo/go/crypto/cipher/ofb_test.go create mode 100644 libgo/go/crypto/elliptic/elliptic.go create mode 100644 libgo/go/crypto/elliptic/elliptic_test.go create mode 100644 libgo/go/crypto/hmac/hmac.go create mode 100644 libgo/go/crypto/hmac/hmac_test.go create mode 100644 libgo/go/crypto/md4/md4.go create mode 100644 libgo/go/crypto/md4/md4_test.go create mode 100644 libgo/go/crypto/md4/md4block.go create mode 100644 libgo/go/crypto/md5/md5.go create mode 100644 libgo/go/crypto/md5/md5_test.go create mode 100644 libgo/go/crypto/md5/md5block.go create mode 100644 libgo/go/crypto/ocsp/ocsp.go create mode 100644 libgo/go/crypto/ocsp/ocsp_test.go create mode 100644 libgo/go/crypto/openpgp/armor/armor.go create mode 100644 libgo/go/crypto/openpgp/armor/armor_test.go create mode 100644 libgo/go/crypto/openpgp/armor/encode.go create mode 100644 libgo/go/crypto/openpgp/error/error.go create mode 100644 libgo/go/crypto/openpgp/s2k/s2k.go create mode 100644 libgo/go/crypto/openpgp/s2k/s2k_test.go create mode 100644 libgo/go/crypto/rand/rand.go create mode 100644 libgo/go/crypto/rand/rand_test.go create mode 100644 libgo/go/crypto/rand/rand_unix.go create mode 100644 libgo/go/crypto/rand/rand_windows.go create mode 100644 libgo/go/crypto/rc4/rc4.go create mode 100644 libgo/go/crypto/rc4/rc4_test.go create mode 100644 libgo/go/crypto/ripemd160/ripemd160.go create mode 100644 libgo/go/crypto/ripemd160/ripemd160_test.go create mode 100644 libgo/go/crypto/ripemd160/ripemd160block.go create mode 100644 libgo/go/crypto/rsa/pkcs1v15.go create mode 100644 libgo/go/crypto/rsa/pkcs1v15_test.go create mode 100644 libgo/go/crypto/rsa/rsa.go create mode 100644 libgo/go/crypto/rsa/rsa_test.go create mode 100644 libgo/go/crypto/sha1/sha1.go create mode 100644 libgo/go/crypto/sha1/sha1_test.go create mode 100644 libgo/go/crypto/sha1/sha1block.go create mode 100644 libgo/go/crypto/sha256/sha256.go create mode 100644 libgo/go/crypto/sha256/sha256_test.go create mode 100644 libgo/go/crypto/sha256/sha256block.go create mode 100644 libgo/go/crypto/sha512/sha512.go create mode 100644 libgo/go/crypto/sha512/sha512_test.go create mode 100644 libgo/go/crypto/sha512/sha512block.go create mode 100644 libgo/go/crypto/subtle/constant_time.go create mode 100644 libgo/go/crypto/subtle/constant_time_test.go create mode 100644 libgo/go/crypto/tls/alert.go create mode 100644 libgo/go/crypto/tls/ca_set.go create mode 100644 libgo/go/crypto/tls/cipher_suites.go create mode 100644 libgo/go/crypto/tls/common.go create mode 100644 libgo/go/crypto/tls/conn.go create mode 100644 libgo/go/crypto/tls/conn_test.go create mode 100644 libgo/go/crypto/tls/generate_cert.go create mode 100644 libgo/go/crypto/tls/handshake_client.go create mode 100644 libgo/go/crypto/tls/handshake_client_test.go create mode 100644 libgo/go/crypto/tls/handshake_messages.go create mode 100644 libgo/go/crypto/tls/handshake_messages_test.go create mode 100644 libgo/go/crypto/tls/handshake_server.go create mode 100644 libgo/go/crypto/tls/handshake_server_test.go create mode 100644 libgo/go/crypto/tls/key_agreement.go create mode 100644 libgo/go/crypto/tls/prf.go create mode 100644 libgo/go/crypto/tls/prf_test.go create mode 100644 libgo/go/crypto/tls/tls.go create mode 100644 libgo/go/crypto/twofish/twofish.go create mode 100644 libgo/go/crypto/twofish/twofish_test.go create mode 100644 libgo/go/crypto/x509/x509.go create mode 100644 libgo/go/crypto/x509/x509_test.go create mode 100644 libgo/go/crypto/xtea/block.go create mode 100644 libgo/go/crypto/xtea/cipher.go create mode 100644 libgo/go/crypto/xtea/xtea_test.go create mode 100644 libgo/go/debug/dwarf/buf.go create mode 100644 libgo/go/debug/dwarf/const.go create mode 100644 libgo/go/debug/dwarf/entry.go create mode 100644 libgo/go/debug/dwarf/open.go create mode 100644 libgo/go/debug/dwarf/testdata/typedef.c create mode 100755 libgo/go/debug/dwarf/testdata/typedef.elf create mode 100644 libgo/go/debug/dwarf/testdata/typedef.macho create mode 100644 libgo/go/debug/dwarf/type.go create mode 100644 libgo/go/debug/dwarf/type_test.go create mode 100644 libgo/go/debug/dwarf/unit.go create mode 100644 libgo/go/debug/elf/elf.go create mode 100644 libgo/go/debug/elf/elf_test.go create mode 100644 libgo/go/debug/elf/file.go create mode 100644 libgo/go/debug/elf/file_test.go create mode 100755 libgo/go/debug/elf/testdata/gcc-386-freebsd-exec create mode 100755 libgo/go/debug/elf/testdata/gcc-amd64-linux-exec create mode 100644 libgo/go/debug/elf/testdata/go-relocation-test-gcc424-x86-64.o create mode 100644 libgo/go/debug/elf/testdata/go-relocation-test-gcc441-x86-64.o create mode 100644 libgo/go/debug/elf/testdata/go-relocation-test-gcc441-x86.o create mode 100644 libgo/go/debug/gosym/pclinetest.h create mode 100644 libgo/go/debug/gosym/pclinetest.s create mode 100644 libgo/go/debug/gosym/pclntab.go create mode 100644 libgo/go/debug/gosym/pclntab_test.go create mode 100644 libgo/go/debug/gosym/symtab.go create mode 100644 libgo/go/debug/macho/file.go create mode 100644 libgo/go/debug/macho/file_test.go create mode 100644 libgo/go/debug/macho/macho.go create mode 100755 libgo/go/debug/macho/testdata/gcc-386-darwin-exec create mode 100755 libgo/go/debug/macho/testdata/gcc-amd64-darwin-exec create mode 100644 libgo/go/debug/macho/testdata/gcc-amd64-darwin-exec-debug create mode 100644 libgo/go/debug/macho/testdata/hello.c create mode 100644 libgo/go/debug/pe/file.go create mode 100644 libgo/go/debug/pe/file_test.go create mode 100644 libgo/go/debug/pe/pe.go create mode 100644 libgo/go/debug/pe/testdata/gcc-386-mingw-exec create mode 100644 libgo/go/debug/pe/testdata/gcc-386-mingw-obj create mode 100644 libgo/go/debug/pe/testdata/hello.c create mode 100644 libgo/go/debug/proc/proc.go create mode 100644 libgo/go/debug/proc/proc_darwin.go create mode 100644 libgo/go/debug/proc/proc_freebsd.go create mode 100644 libgo/go/debug/proc/proc_linux.go create mode 100644 libgo/go/debug/proc/proc_rtems.go create mode 100644 libgo/go/debug/proc/proc_solaris.go create mode 100644 libgo/go/debug/proc/proc_windows.go create mode 100644 libgo/go/debug/proc/ptrace-nptl.txt create mode 100644 libgo/go/debug/proc/regs_darwin_386.go create mode 100644 libgo/go/debug/proc/regs_darwin_amd64.go create mode 100644 libgo/go/debug/proc/regs_freebsd_386.go create mode 100644 libgo/go/debug/proc/regs_freebsd_amd64.go create mode 100644 libgo/go/debug/proc/regs_linux_386.go create mode 100644 libgo/go/debug/proc/regs_linux_amd64.go create mode 100644 libgo/go/debug/proc/regs_linux_arm.go create mode 100644 libgo/go/debug/proc/regs_windows_386.go create mode 100644 libgo/go/debug/proc/regs_windows_amd64.go create mode 100644 libgo/go/ebnf/ebnf.go create mode 100644 libgo/go/ebnf/ebnf_test.go create mode 100644 libgo/go/ebnf/parser.go create mode 100644 libgo/go/encoding/ascii85/ascii85.go create mode 100644 libgo/go/encoding/ascii85/ascii85_test.go create mode 100644 libgo/go/encoding/base32/base32.go create mode 100644 libgo/go/encoding/base32/base32_test.go create mode 100644 libgo/go/encoding/base64/base64.go create mode 100644 libgo/go/encoding/base64/base64_test.go create mode 100644 libgo/go/encoding/binary/binary.go create mode 100644 libgo/go/encoding/binary/binary_test.go create mode 100644 libgo/go/encoding/git85/git.go create mode 100644 libgo/go/encoding/git85/git_test.go create mode 100644 libgo/go/encoding/hex/hex.go create mode 100644 libgo/go/encoding/hex/hex_test.go create mode 100644 libgo/go/encoding/line/line.go create mode 100644 libgo/go/encoding/line/line_test.go create mode 100644 libgo/go/encoding/pem/pem.go create mode 100644 libgo/go/encoding/pem/pem_test.go create mode 100644 libgo/go/exec/exec.go create mode 100644 libgo/go/exec/exec_test.go create mode 100644 libgo/go/exec/lp_unix.go create mode 100644 libgo/go/exec/lp_windows.go create mode 100644 libgo/go/exp/README create mode 100644 libgo/go/exp/datafmt/datafmt.go create mode 100644 libgo/go/exp/datafmt/datafmt_test.go create mode 100644 libgo/go/exp/datafmt/parser.go create mode 100644 libgo/go/exp/draw/draw.go create mode 100644 libgo/go/exp/draw/draw_test.go create mode 100644 libgo/go/exp/draw/event.go create mode 100644 libgo/go/exp/draw/x11/auth.go create mode 100644 libgo/go/exp/draw/x11/conn.go create mode 100644 libgo/go/exp/eval/abort.go create mode 100644 libgo/go/exp/eval/bridge.go create mode 100644 libgo/go/exp/eval/compiler.go create mode 100644 libgo/go/exp/eval/eval_test.go create mode 100644 libgo/go/exp/eval/expr.go create mode 100644 libgo/go/exp/eval/expr1.go create mode 100644 libgo/go/exp/eval/expr_test.go create mode 100644 libgo/go/exp/eval/func.go create mode 100644 libgo/go/exp/eval/scope.go create mode 100644 libgo/go/exp/eval/stmt.go create mode 100644 libgo/go/exp/eval/stmt_test.go create mode 100644 libgo/go/exp/eval/type.go create mode 100644 libgo/go/exp/eval/typec.go create mode 100644 libgo/go/exp/eval/value.go create mode 100644 libgo/go/exp/eval/world.go create mode 100644 libgo/go/exp/ogle/abort.go create mode 100644 libgo/go/exp/ogle/arch.go create mode 100644 libgo/go/exp/ogle/cmd.go create mode 100644 libgo/go/exp/ogle/event.go create mode 100644 libgo/go/exp/ogle/frame.go create mode 100644 libgo/go/exp/ogle/goroutine.go create mode 100644 libgo/go/exp/ogle/main.go create mode 100644 libgo/go/exp/ogle/process.go create mode 100644 libgo/go/exp/ogle/rruntime.go create mode 100644 libgo/go/exp/ogle/rtype.go create mode 100644 libgo/go/exp/ogle/rvalue.go create mode 100644 libgo/go/exp/ogle/vars.go create mode 100644 libgo/go/expvar/expvar.go create mode 100644 libgo/go/expvar/expvar_test.go create mode 100644 libgo/go/flag/export_test.go create mode 100644 libgo/go/flag/flag.go create mode 100644 libgo/go/flag/flag_test.go create mode 100644 libgo/go/fmt/doc.go create mode 100644 libgo/go/fmt/fmt_test.go create mode 100644 libgo/go/fmt/format.go create mode 100644 libgo/go/fmt/print.go create mode 100644 libgo/go/fmt/scan.go create mode 100644 libgo/go/fmt/scan_test.go create mode 100644 libgo/go/fmt/stringer_test.go create mode 100644 libgo/go/go/ast/ast.go create mode 100644 libgo/go/go/ast/filter.go create mode 100644 libgo/go/go/ast/print.go create mode 100644 libgo/go/go/ast/scope.go create mode 100644 libgo/go/go/ast/walk.go create mode 100644 libgo/go/go/doc/comment.go create mode 100644 libgo/go/go/doc/doc.go create mode 100644 libgo/go/go/parser/interface.go create mode 100644 libgo/go/go/parser/parser.go create mode 100644 libgo/go/go/parser/parser_test.go create mode 100644 libgo/go/go/printer/nodes.go create mode 100644 libgo/go/go/printer/printer.go create mode 100644 libgo/go/go/printer/printer_test.go create mode 100644 libgo/go/go/printer/testdata/comments.golden create mode 100644 libgo/go/go/printer/testdata/comments.input create mode 100644 libgo/go/go/printer/testdata/comments.x create mode 100644 libgo/go/go/printer/testdata/declarations.golden create mode 100644 libgo/go/go/printer/testdata/declarations.input create mode 100644 libgo/go/go/printer/testdata/empty.golden create mode 100644 libgo/go/go/printer/testdata/empty.input create mode 100644 libgo/go/go/printer/testdata/expressions.golden create mode 100644 libgo/go/go/printer/testdata/expressions.input create mode 100644 libgo/go/go/printer/testdata/expressions.raw create mode 100644 libgo/go/go/printer/testdata/linebreaks.golden create mode 100644 libgo/go/go/printer/testdata/linebreaks.input create mode 100644 libgo/go/go/printer/testdata/statements.golden create mode 100644 libgo/go/go/printer/testdata/statements.input create mode 100644 libgo/go/go/scanner/errors.go create mode 100644 libgo/go/go/scanner/scanner.go create mode 100644 libgo/go/go/scanner/scanner_test.go create mode 100644 libgo/go/go/token/position.go create mode 100644 libgo/go/go/token/position_test.go create mode 100644 libgo/go/go/token/token.go create mode 100644 libgo/go/go/typechecker/scope.go create mode 100644 libgo/go/go/typechecker/testdata/test0.go create mode 100644 libgo/go/go/typechecker/testdata/test1.go create mode 100644 libgo/go/go/typechecker/testdata/test3.go create mode 100644 libgo/go/go/typechecker/testdata/test4.go create mode 100644 libgo/go/go/typechecker/typechecker.go create mode 100644 libgo/go/go/typechecker/typechecker_test.go create mode 100644 libgo/go/go/typechecker/universe.go create mode 100644 libgo/go/gob/codec_test.go create mode 100644 libgo/go/gob/decode.go create mode 100644 libgo/go/gob/decoder.go create mode 100644 libgo/go/gob/doc.go create mode 100644 libgo/go/gob/encode.go create mode 100644 libgo/go/gob/encoder.go create mode 100644 libgo/go/gob/encoder_test.go create mode 100644 libgo/go/gob/error.go create mode 100644 libgo/go/gob/type.go create mode 100644 libgo/go/gob/type_test.go create mode 100644 libgo/go/hash/adler32/adler32.go create mode 100644 libgo/go/hash/adler32/adler32_test.go create mode 100644 libgo/go/hash/crc32/crc32.go create mode 100644 libgo/go/hash/crc32/crc32_test.go create mode 100644 libgo/go/hash/crc64/crc64.go create mode 100644 libgo/go/hash/crc64/crc64_test.go create mode 100644 libgo/go/hash/hash.go create mode 100644 libgo/go/html/doc.go create mode 100644 libgo/go/html/entity.go create mode 100644 libgo/go/html/entity_test.go create mode 100644 libgo/go/html/escape.go create mode 100644 libgo/go/html/parse.go create mode 100644 libgo/go/html/parse_test.go create mode 100644 libgo/go/html/testdata/webkit/README create mode 100644 libgo/go/html/testdata/webkit/comments01.dat create mode 100644 libgo/go/html/testdata/webkit/doctype01.dat create mode 100644 libgo/go/html/testdata/webkit/dom2string.js create mode 100644 libgo/go/html/testdata/webkit/entities01.dat create mode 100644 libgo/go/html/testdata/webkit/entities02.dat create mode 100644 libgo/go/html/testdata/webkit/scriptdata01.dat create mode 100644 libgo/go/html/testdata/webkit/tests1.dat create mode 100644 libgo/go/html/testdata/webkit/tests10.dat create mode 100644 libgo/go/html/testdata/webkit/tests11.dat create mode 100644 libgo/go/html/testdata/webkit/tests12.dat create mode 100644 libgo/go/html/testdata/webkit/tests13.dat create mode 100644 libgo/go/html/testdata/webkit/tests14.dat create mode 100644 libgo/go/html/testdata/webkit/tests15.dat create mode 100644 libgo/go/html/testdata/webkit/tests16.dat create mode 100644 libgo/go/html/testdata/webkit/tests2.dat create mode 100644 libgo/go/html/testdata/webkit/tests3.dat create mode 100644 libgo/go/html/testdata/webkit/tests4.dat create mode 100644 libgo/go/html/testdata/webkit/tests5.dat create mode 100644 libgo/go/html/testdata/webkit/tests6.dat create mode 100644 libgo/go/html/testdata/webkit/tests7.dat create mode 100644 libgo/go/html/testdata/webkit/tests8.dat create mode 100644 libgo/go/html/testdata/webkit/tests9.dat create mode 100644 libgo/go/html/testdata/webkit/webkit01.dat create mode 100644 libgo/go/html/token.go create mode 100644 libgo/go/html/token_test.go create mode 100644 libgo/go/http/chunked.go create mode 100644 libgo/go/http/client.go create mode 100644 libgo/go/http/client_test.go create mode 100644 libgo/go/http/dump.go create mode 100644 libgo/go/http/fs.go create mode 100644 libgo/go/http/fs_test.go create mode 100644 libgo/go/http/lex.go create mode 100644 libgo/go/http/lex_test.go create mode 100644 libgo/go/http/persist.go create mode 100644 libgo/go/http/pprof/pprof.go create mode 100644 libgo/go/http/readrequest_test.go create mode 100644 libgo/go/http/request.go create mode 100644 libgo/go/http/request_test.go create mode 100644 libgo/go/http/requestwrite_test.go create mode 100644 libgo/go/http/response.go create mode 100644 libgo/go/http/response_test.go create mode 100644 libgo/go/http/responsewrite_test.go create mode 100644 libgo/go/http/serve_test.go create mode 100644 libgo/go/http/server.go create mode 100644 libgo/go/http/status.go create mode 100644 libgo/go/http/testdata/file create mode 100644 libgo/go/http/transfer.go create mode 100644 libgo/go/http/url.go create mode 100644 libgo/go/http/url_test.go create mode 100644 libgo/go/image/color.go create mode 100644 libgo/go/image/format.go create mode 100644 libgo/go/image/geom.go create mode 100644 libgo/go/image/image.go create mode 100644 libgo/go/image/jpeg/huffman.go create mode 100644 libgo/go/image/jpeg/idct.go create mode 100644 libgo/go/image/jpeg/reader.go create mode 100644 libgo/go/image/names.go create mode 100644 libgo/go/image/png/reader.go create mode 100644 libgo/go/image/png/reader_test.go create mode 100644 libgo/go/image/png/testdata/pngsuite/README create mode 100644 libgo/go/image/png/testdata/pngsuite/README.original create mode 100644 libgo/go/image/png/testdata/pngsuite/basn0g01.png create mode 100644 libgo/go/image/png/testdata/pngsuite/basn0g01.sng create mode 100644 libgo/go/image/png/testdata/pngsuite/basn0g02.png create mode 100644 libgo/go/image/png/testdata/pngsuite/basn0g02.sng create mode 100644 libgo/go/image/png/testdata/pngsuite/basn0g04.png create mode 100644 libgo/go/image/png/testdata/pngsuite/basn0g04.sng create mode 100644 libgo/go/image/png/testdata/pngsuite/basn0g08.png create mode 100644 libgo/go/image/png/testdata/pngsuite/basn0g08.sng create mode 100644 libgo/go/image/png/testdata/pngsuite/basn0g16.png create mode 100644 libgo/go/image/png/testdata/pngsuite/basn0g16.sng create mode 100644 libgo/go/image/png/testdata/pngsuite/basn2c08.png create mode 100644 libgo/go/image/png/testdata/pngsuite/basn2c08.sng create mode 100644 libgo/go/image/png/testdata/pngsuite/basn2c16.png create mode 100644 libgo/go/image/png/testdata/pngsuite/basn2c16.sng create mode 100644 libgo/go/image/png/testdata/pngsuite/basn3p01.png create mode 100644 libgo/go/image/png/testdata/pngsuite/basn3p01.sng create mode 100644 libgo/go/image/png/testdata/pngsuite/basn3p02.png create mode 100644 libgo/go/image/png/testdata/pngsuite/basn3p02.sng create mode 100644 libgo/go/image/png/testdata/pngsuite/basn3p04.png create mode 100644 libgo/go/image/png/testdata/pngsuite/basn3p04.sng create mode 100644 libgo/go/image/png/testdata/pngsuite/basn3p08.png create mode 100644 libgo/go/image/png/testdata/pngsuite/basn3p08.sng create mode 100644 libgo/go/image/png/testdata/pngsuite/basn4a08.png create mode 100644 libgo/go/image/png/testdata/pngsuite/basn4a08.sng create mode 100644 libgo/go/image/png/testdata/pngsuite/basn4a16.png create mode 100644 libgo/go/image/png/testdata/pngsuite/basn4a16.sng create mode 100644 libgo/go/image/png/testdata/pngsuite/basn6a08.png create mode 100644 libgo/go/image/png/testdata/pngsuite/basn6a08.sng create mode 100644 libgo/go/image/png/testdata/pngsuite/basn6a16.png create mode 100644 libgo/go/image/png/testdata/pngsuite/basn6a16.sng create mode 100644 libgo/go/image/png/writer.go create mode 100644 libgo/go/image/png/writer_test.go create mode 100644 libgo/go/index/suffixarray/qsufsort.go create mode 100644 libgo/go/index/suffixarray/suffixarray.go create mode 100644 libgo/go/index/suffixarray/suffixarray_test.go create mode 100644 libgo/go/io/io.go create mode 100644 libgo/go/io/io_test.go create mode 100644 libgo/go/io/ioutil/ioutil.go create mode 100644 libgo/go/io/ioutil/ioutil_test.go create mode 100644 libgo/go/io/ioutil/tempfile.go create mode 100644 libgo/go/io/ioutil/tempfile_test.go create mode 100644 libgo/go/io/multi.go create mode 100644 libgo/go/io/multi_test.go create mode 100644 libgo/go/io/pipe.go create mode 100644 libgo/go/io/pipe_test.go create mode 100644 libgo/go/json/decode.go create mode 100644 libgo/go/json/decode_test.go create mode 100644 libgo/go/json/encode.go create mode 100644 libgo/go/json/indent.go create mode 100644 libgo/go/json/scanner.go create mode 100644 libgo/go/json/scanner_test.go create mode 100644 libgo/go/json/stream.go create mode 100644 libgo/go/json/stream_test.go create mode 100644 libgo/go/log/log.go create mode 100644 libgo/go/log/log_test.go create mode 100644 libgo/go/math/acosh.go create mode 100644 libgo/go/math/all_test.go create mode 100644 libgo/go/math/asin.go create mode 100644 libgo/go/math/asinh.go create mode 100644 libgo/go/math/atan.go create mode 100644 libgo/go/math/atan2.go create mode 100644 libgo/go/math/atanh.go create mode 100644 libgo/go/math/bits.go create mode 100644 libgo/go/math/cbrt.go create mode 100644 libgo/go/math/const.go create mode 100644 libgo/go/math/copysign.go create mode 100644 libgo/go/math/erf.go create mode 100644 libgo/go/math/exp.go create mode 100644 libgo/go/math/exp2.go create mode 100644 libgo/go/math/exp_port.go create mode 100644 libgo/go/math/exp_test.go create mode 100644 libgo/go/math/expm1.go create mode 100644 libgo/go/math/fabs.go create mode 100644 libgo/go/math/fdim.go create mode 100644 libgo/go/math/floor.go create mode 100644 libgo/go/math/fmod.go create mode 100644 libgo/go/math/frexp.go create mode 100644 libgo/go/math/gamma.go create mode 100644 libgo/go/math/hypot.go create mode 100644 libgo/go/math/hypot_port.go create mode 100644 libgo/go/math/hypot_test.go create mode 100644 libgo/go/math/j0.go create mode 100644 libgo/go/math/j1.go create mode 100644 libgo/go/math/jn.go create mode 100644 libgo/go/math/ldexp.go create mode 100644 libgo/go/math/lgamma.go create mode 100644 libgo/go/math/log.go create mode 100644 libgo/go/math/log10.go create mode 100644 libgo/go/math/log10_decl.go create mode 100644 libgo/go/math/log1p.go create mode 100644 libgo/go/math/logb.go create mode 100644 libgo/go/math/modf.go create mode 100644 libgo/go/math/nextafter.go create mode 100644 libgo/go/math/pow.go create mode 100644 libgo/go/math/pow10.go create mode 100644 libgo/go/math/remainder.go create mode 100644 libgo/go/math/signbit.go create mode 100644 libgo/go/math/sin.go create mode 100644 libgo/go/math/sincos.go create mode 100644 libgo/go/math/sinh.go create mode 100644 libgo/go/math/sqrt.go create mode 100644 libgo/go/math/sqrt_decl.go create mode 100644 libgo/go/math/sqrt_port.go create mode 100644 libgo/go/math/sqrt_test.go create mode 100644 libgo/go/math/tan.go create mode 100644 libgo/go/math/tanh.go create mode 100644 libgo/go/math/unsafe.go create mode 100644 libgo/go/mime/grammar.go create mode 100644 libgo/go/mime/mediatype.go create mode 100644 libgo/go/mime/mediatype_test.go create mode 100644 libgo/go/mime/mime_test.go create mode 100644 libgo/go/mime/multipart/multipart.go create mode 100644 libgo/go/mime/multipart/multipart_test.go create mode 100644 libgo/go/mime/test.types create mode 100644 libgo/go/mime/type.go create mode 100644 libgo/go/net/dial.go create mode 100644 libgo/go/net/dialgoogle_test.go create mode 100644 libgo/go/net/dict/dict.go create mode 100644 libgo/go/net/dnsclient.go create mode 100644 libgo/go/net/dnsconfig.go create mode 100644 libgo/go/net/dnsmsg.go create mode 100644 libgo/go/net/dnsname_test.go create mode 100644 libgo/go/net/fd.go create mode 100644 libgo/go/net/fd_linux.go create mode 100644 libgo/go/net/fd_rtems.go create mode 100644 libgo/go/net/fd_windows.go create mode 100644 libgo/go/net/hosts.go create mode 100644 libgo/go/net/hosts_test.go create mode 100644 libgo/go/net/hosts_testdata create mode 100644 libgo/go/net/ip.go create mode 100644 libgo/go/net/ip_test.go create mode 100644 libgo/go/net/ipraw_test.go create mode 100644 libgo/go/net/iprawsock.go create mode 100644 libgo/go/net/ipsock.go create mode 100644 libgo/go/net/net.go create mode 100644 libgo/go/net/net_test.go create mode 100644 libgo/go/net/newpollserver.go create mode 100644 libgo/go/net/newpollserver_rtems.go create mode 100644 libgo/go/net/parse.go create mode 100644 libgo/go/net/parse_test.go create mode 100644 libgo/go/net/pipe.go create mode 100644 libgo/go/net/pipe_test.go create mode 100644 libgo/go/net/port.go create mode 100644 libgo/go/net/port_test.go create mode 100644 libgo/go/net/resolv_windows.go create mode 100644 libgo/go/net/server_test.go create mode 100644 libgo/go/net/sock.go create mode 100644 libgo/go/net/srv_test.go create mode 100644 libgo/go/net/tcpsock.go create mode 100644 libgo/go/net/textproto/pipeline.go create mode 100644 libgo/go/net/textproto/reader.go create mode 100644 libgo/go/net/textproto/reader_test.go create mode 100644 libgo/go/net/textproto/textproto.go create mode 100644 libgo/go/net/textproto/writer.go create mode 100644 libgo/go/net/textproto/writer_test.go create mode 100644 libgo/go/net/timeout_test.go create mode 100644 libgo/go/net/udpsock.go create mode 100644 libgo/go/net/unixsock.go create mode 100644 libgo/go/netchan/common.go create mode 100644 libgo/go/netchan/export.go create mode 100644 libgo/go/netchan/import.go create mode 100644 libgo/go/netchan/netchan_test.go create mode 100644 libgo/go/os/dir.go create mode 100644 libgo/go/os/dir_largefile.go create mode 100644 libgo/go/os/dir_regfile.go create mode 100644 libgo/go/os/env.go create mode 100644 libgo/go/os/env_test.go create mode 100644 libgo/go/os/env_unix.go create mode 100644 libgo/go/os/env_windows.go create mode 100644 libgo/go/os/error.go create mode 100644 libgo/go/os/exec.go create mode 100644 libgo/go/os/file.go create mode 100644 libgo/go/os/file_unix.go create mode 100644 libgo/go/os/getwd.go create mode 100644 libgo/go/os/inotify/inotify_linux.go create mode 100644 libgo/go/os/inotify/inotify_linux_test.go create mode 100644 libgo/go/os/os_test.go create mode 100644 libgo/go/os/path.go create mode 100644 libgo/go/os/path_test.go create mode 100644 libgo/go/os/proc.go create mode 100644 libgo/go/os/signal/mkunix.sh create mode 100644 libgo/go/os/signal/signal.go create mode 100644 libgo/go/os/signal/signal_test.go create mode 100644 libgo/go/os/stat.go create mode 100644 libgo/go/os/sys_bsd.go create mode 100644 libgo/go/os/sys_linux.go create mode 100644 libgo/go/os/sys_uname.go create mode 100644 libgo/go/os/time.go create mode 100644 libgo/go/os/types.go create mode 100644 libgo/go/patch/apply.go create mode 100644 libgo/go/patch/git.go create mode 100644 libgo/go/patch/patch.go create mode 100644 libgo/go/patch/patch_test.go create mode 100644 libgo/go/patch/textdiff.go create mode 100644 libgo/go/path/match.go create mode 100644 libgo/go/path/match_test.go create mode 100644 libgo/go/path/path.go create mode 100644 libgo/go/path/path_test.go create mode 100644 libgo/go/path/path_unix.go create mode 100644 libgo/go/path/path_windows.go create mode 100644 libgo/go/rand/exp.go create mode 100644 libgo/go/rand/normal.go create mode 100644 libgo/go/rand/rand.go create mode 100644 libgo/go/rand/rand_test.go create mode 100644 libgo/go/rand/rng.go create mode 100644 libgo/go/rand/zipf.go create mode 100644 libgo/go/reflect/all_test.go create mode 100644 libgo/go/reflect/deepequal.go create mode 100644 libgo/go/reflect/tostring_test.go create mode 100644 libgo/go/reflect/type.go create mode 100644 libgo/go/reflect/value.go create mode 100644 libgo/go/regexp/all_test.go create mode 100644 libgo/go/regexp/find_test.go create mode 100644 libgo/go/regexp/regexp.go create mode 100644 libgo/go/rpc/client.go create mode 100644 libgo/go/rpc/debug.go create mode 100644 libgo/go/rpc/jsonrpc/all_test.go create mode 100644 libgo/go/rpc/jsonrpc/client.go create mode 100644 libgo/go/rpc/jsonrpc/server.go create mode 100644 libgo/go/rpc/server.go create mode 100644 libgo/go/rpc/server_test.go create mode 100644 libgo/go/runtime/chan_defs.go create mode 100644 libgo/go/runtime/debug.go create mode 100644 libgo/go/runtime/debug/stack.go create mode 100644 libgo/go/runtime/debug/stack_test.go create mode 100644 libgo/go/runtime/error.go create mode 100644 libgo/go/runtime/export_test.go create mode 100644 libgo/go/runtime/extern.go create mode 100644 libgo/go/runtime/hashmap_defs.go create mode 100644 libgo/go/runtime/iface_defs.go create mode 100644 libgo/go/runtime/malloc_defs.go create mode 100644 libgo/go/runtime/mheapmap32_defs.go create mode 100644 libgo/go/runtime/mheapmap64_defs.go create mode 100644 libgo/go/runtime/pprof/pprof.go create mode 100644 libgo/go/runtime/runtime_defs.go create mode 100644 libgo/go/runtime/sig.go create mode 100644 libgo/go/runtime/softfloat64.go create mode 100644 libgo/go/runtime/softfloat64_test.go create mode 100644 libgo/go/runtime/type.go create mode 100644 libgo/go/scanner/scanner.go create mode 100644 libgo/go/scanner/scanner_test.go create mode 100644 libgo/go/smtp/auth.go create mode 100644 libgo/go/smtp/smtp.go create mode 100644 libgo/go/smtp/smtp_test.go create mode 100644 libgo/go/sort/search.go create mode 100644 libgo/go/sort/search_test.go create mode 100644 libgo/go/sort/sort.go create mode 100644 libgo/go/sort/sort_test.go create mode 100644 libgo/go/strconv/atob.go create mode 100644 libgo/go/strconv/atob_test.go create mode 100644 libgo/go/strconv/atof.go create mode 100644 libgo/go/strconv/atof_test.go create mode 100644 libgo/go/strconv/atoi.go create mode 100644 libgo/go/strconv/atoi_test.go create mode 100644 libgo/go/strconv/decimal.go create mode 100644 libgo/go/strconv/decimal_test.go create mode 100644 libgo/go/strconv/fp_test.go create mode 100644 libgo/go/strconv/ftoa.go create mode 100644 libgo/go/strconv/ftoa_test.go create mode 100644 libgo/go/strconv/internal_test.go create mode 100644 libgo/go/strconv/itoa.go create mode 100644 libgo/go/strconv/itoa_test.go create mode 100644 libgo/go/strconv/quote.go create mode 100644 libgo/go/strconv/quote_test.go create mode 100644 libgo/go/strconv/testfp.txt create mode 100644 libgo/go/strings/reader.go create mode 100644 libgo/go/strings/strings.go create mode 100644 libgo/go/strings/strings_test.go create mode 100644 libgo/go/sync/cas.c create mode 100644 libgo/go/sync/mutex.go create mode 100644 libgo/go/sync/mutex_test.go create mode 100644 libgo/go/sync/once.go create mode 100644 libgo/go/sync/once_test.go create mode 100644 libgo/go/sync/rwmutex.go create mode 100644 libgo/go/sync/rwmutex_test.go create mode 100644 libgo/go/sync/xadd_test.go create mode 100644 libgo/go/syslog/syslog.go create mode 100644 libgo/go/syslog/syslog_c.c create mode 100644 libgo/go/syslog/syslog_solaris.go create mode 100644 libgo/go/syslog/syslog_test.go create mode 100644 libgo/go/syslog/syslog_unix.go create mode 100644 libgo/go/tabwriter/tabwriter.go create mode 100644 libgo/go/tabwriter/tabwriter_test.go create mode 100644 libgo/go/template/format.go create mode 100644 libgo/go/template/template.go create mode 100644 libgo/go/template/template_test.go create mode 100644 libgo/go/testing/benchmark.go create mode 100644 libgo/go/testing/iotest/logger.go create mode 100644 libgo/go/testing/iotest/reader.go create mode 100644 libgo/go/testing/iotest/writer.go create mode 100644 libgo/go/testing/quick/quick.go create mode 100644 libgo/go/testing/quick/quick_test.go create mode 100644 libgo/go/testing/script/script.go create mode 100644 libgo/go/testing/script/script_test.go create mode 100644 libgo/go/testing/testing.go create mode 100644 libgo/go/time/format.go create mode 100644 libgo/go/time/sleep.go create mode 100644 libgo/go/time/sleep_test.go create mode 100644 libgo/go/time/tick.go create mode 100644 libgo/go/time/tick_test.go create mode 100644 libgo/go/time/time.go create mode 100644 libgo/go/time/time_test.go create mode 100644 libgo/go/time/zoneinfo_unix.go create mode 100644 libgo/go/time/zoneinfo_windows.go create mode 100644 libgo/go/try/try.go create mode 100644 libgo/go/try/try_test.go create mode 100644 libgo/go/unicode/casetables.go create mode 100644 libgo/go/unicode/digit.go create mode 100644 libgo/go/unicode/digit_test.go create mode 100644 libgo/go/unicode/letter.go create mode 100644 libgo/go/unicode/letter_test.go create mode 100644 libgo/go/unicode/script_test.go create mode 100644 libgo/go/unicode/tables.go create mode 100644 libgo/go/utf16/utf16.go create mode 100644 libgo/go/utf16/utf16_test.go create mode 100644 libgo/go/utf8/string.go create mode 100644 libgo/go/utf8/string_test.go create mode 100644 libgo/go/utf8/utf8.go create mode 100644 libgo/go/utf8/utf8_test.go create mode 100644 libgo/go/websocket/client.go create mode 100644 libgo/go/websocket/server.go create mode 100644 libgo/go/websocket/websocket.go create mode 100644 libgo/go/websocket/websocket_test.go create mode 100644 libgo/go/xml/embed_test.go create mode 100644 libgo/go/xml/read.go create mode 100644 libgo/go/xml/read_test.go create mode 100644 libgo/go/xml/xml.go create mode 100644 libgo/go/xml/xml_test.go (limited to 'libgo/go') diff --git a/libgo/go/archive/tar/common.go b/libgo/go/archive/tar/common.go new file mode 100644 index 000000000..5b781ff3d --- /dev/null +++ b/libgo/go/archive/tar/common.go @@ -0,0 +1,75 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The tar package implements access to tar archives. +// It aims to cover most of the variations, including those produced +// by GNU and BSD tars. +// +// References: +// http://www.freebsd.org/cgi/man.cgi?query=tar&sektion=5 +// http://www.gnu.org/software/tar/manual/html_node/Standard.html +package tar + +const ( + blockSize = 512 + + // Types + TypeReg = '0' + TypeRegA = '\x00' + TypeLink = '1' + TypeSymlink = '2' + TypeChar = '3' + TypeBlock = '4' + TypeDir = '5' + TypeFifo = '6' + TypeCont = '7' + TypeXHeader = 'x' + TypeXGlobalHeader = 'g' +) + +// A Header represents a single header in a tar archive. +// Some fields may not be populated. +type Header struct { + Name string + Mode int64 + Uid int + Gid int + Size int64 + Mtime int64 + Typeflag byte + Linkname string + Uname string + Gname string + Devmajor int64 + Devminor int64 + Atime int64 + Ctime int64 +} + +var zeroBlock = make([]byte, blockSize) + +// POSIX specifies a sum of the unsigned byte values, but the Sun tar uses signed byte values. +// We compute and return both. +func checksum(header []byte) (unsigned int64, signed int64) { + for i := 0; i < len(header); i++ { + if i == 148 { + // The chksum field (header[148:156]) is special: it should be treated as space bytes. + unsigned += ' ' * 8 + signed += ' ' * 8 + i += 7 + continue + } + unsigned += int64(header[i]) + signed += int64(int8(header[i])) + } + return +} + +type slicer []byte + +func (sp *slicer) next(n int) (b []byte) { + s := *sp + b, *sp = s[0:n], s[n:] + return +} diff --git a/libgo/go/archive/tar/reader.go b/libgo/go/archive/tar/reader.go new file mode 100644 index 000000000..35a15f74b --- /dev/null +++ b/libgo/go/archive/tar/reader.go @@ -0,0 +1,226 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tar + +// TODO(dsymonds): +// - pax extensions + +import ( + "bytes" + "io" + "os" + "strconv" +) + +var ( + HeaderError os.Error = os.ErrorString("invalid tar header") +) + +// A Reader provides sequential access to the contents of a tar archive. +// A tar archive consists of a sequence of files. +// The Next method advances to the next file in the archive (including the first), +// and then it can be treated as an io.Reader to access the file's data. +// +// Example: +// tr := tar.NewReader(r) +// for { +// hdr, err := tr.Next() +// if err != nil { +// // handle error +// } +// if hdr == nil { +// // end of tar archive +// break +// } +// io.Copy(data, tr) +// } +type Reader struct { + r io.Reader + err os.Error + nb int64 // number of unread bytes for current file entry + pad int64 // amount of padding (ignored) after current file entry +} + +// NewReader creates a new Reader reading from r. +func NewReader(r io.Reader) *Reader { return &Reader{r: r} } + +// Next advances to the next entry in the tar archive. +func (tr *Reader) Next() (*Header, os.Error) { + var hdr *Header + if tr.err == nil { + tr.skipUnread() + } + if tr.err == nil { + hdr = tr.readHeader() + } + return hdr, tr.err +} + +// Parse bytes as a NUL-terminated C-style string. +// If a NUL byte is not found then the whole slice is returned as a string. +func cString(b []byte) string { + n := 0 + for n < len(b) && b[n] != 0 { + n++ + } + return string(b[0:n]) +} + +func (tr *Reader) octal(b []byte) int64 { + // Removing leading spaces. + for len(b) > 0 && b[0] == ' ' { + b = b[1:] + } + // Removing trailing NULs and spaces. + for len(b) > 0 && (b[len(b)-1] == ' ' || b[len(b)-1] == '\x00') { + b = b[0 : len(b)-1] + } + x, err := strconv.Btoui64(cString(b), 8) + if err != nil { + tr.err = err + } + return int64(x) +} + +type ignoreWriter struct{} + +func (ignoreWriter) Write(b []byte) (n int, err os.Error) { + return len(b), nil +} + +// Skip any unread bytes in the existing file entry, as well as any alignment padding. +func (tr *Reader) skipUnread() { + nr := tr.nb + tr.pad // number of bytes to skip + tr.nb, tr.pad = 0, 0 + if sr, ok := tr.r.(io.Seeker); ok { + if _, err := sr.Seek(nr, 1); err == nil { + return + } + } + _, tr.err = io.Copyn(ignoreWriter{}, tr.r, nr) +} + +func (tr *Reader) verifyChecksum(header []byte) bool { + if tr.err != nil { + return false + } + + given := tr.octal(header[148:156]) + unsigned, signed := checksum(header) + return given == unsigned || given == signed +} + +func (tr *Reader) readHeader() *Header { + header := make([]byte, blockSize) + if _, tr.err = io.ReadFull(tr.r, header); tr.err != nil { + return nil + } + + // Two blocks of zero bytes marks the end of the archive. + if bytes.Equal(header, zeroBlock[0:blockSize]) { + if _, tr.err = io.ReadFull(tr.r, header); tr.err != nil { + return nil + } + if bytes.Equal(header, zeroBlock[0:blockSize]) { + tr.err = os.EOF + } else { + tr.err = HeaderError // zero block and then non-zero block + } + return nil + } + + if !tr.verifyChecksum(header) { + tr.err = HeaderError + return nil + } + + // Unpack + hdr := new(Header) + s := slicer(header) + + hdr.Name = cString(s.next(100)) + hdr.Mode = tr.octal(s.next(8)) + hdr.Uid = int(tr.octal(s.next(8))) + hdr.Gid = int(tr.octal(s.next(8))) + hdr.Size = tr.octal(s.next(12)) + hdr.Mtime = tr.octal(s.next(12)) + s.next(8) // chksum + hdr.Typeflag = s.next(1)[0] + hdr.Linkname = cString(s.next(100)) + + // The remainder of the header depends on the value of magic. + // The original (v7) version of tar had no explicit magic field, + // so its magic bytes, like the rest of the block, are NULs. + magic := string(s.next(8)) // contains version field as well. + var format string + switch magic { + case "ustar\x0000": // POSIX tar (1003.1-1988) + if string(header[508:512]) == "tar\x00" { + format = "star" + } else { + format = "posix" + } + case "ustar \x00": // old GNU tar + format = "gnu" + } + + switch format { + case "posix", "gnu", "star": + hdr.Uname = cString(s.next(32)) + hdr.Gname = cString(s.next(32)) + devmajor := s.next(8) + devminor := s.next(8) + if hdr.Typeflag == TypeChar || hdr.Typeflag == TypeBlock { + hdr.Devmajor = tr.octal(devmajor) + hdr.Devminor = tr.octal(devminor) + } + var prefix string + switch format { + case "posix", "gnu": + prefix = cString(s.next(155)) + case "star": + prefix = cString(s.next(131)) + hdr.Atime = tr.octal(s.next(12)) + hdr.Ctime = tr.octal(s.next(12)) + } + if len(prefix) > 0 { + hdr.Name = prefix + "/" + hdr.Name + } + } + + if tr.err != nil { + tr.err = HeaderError + return nil + } + + // Maximum value of hdr.Size is 64 GB (12 octal digits), + // so there's no risk of int64 overflowing. + tr.nb = int64(hdr.Size) + tr.pad = -tr.nb & (blockSize - 1) // blockSize is a power of two + + return hdr +} + +// Read reads from the current entry in the tar archive. +// It returns 0, os.EOF when it reaches the end of that entry, +// until Next is called to advance to the next entry. +func (tr *Reader) Read(b []byte) (n int, err os.Error) { + if tr.nb == 0 { + // file consumed + return 0, os.EOF + } + + if int64(len(b)) > tr.nb { + b = b[0:tr.nb] + } + n, err = tr.r.Read(b) + tr.nb -= int64(n) + + if err == os.EOF && tr.nb > 0 { + err = io.ErrUnexpectedEOF + } + tr.err = err + return +} diff --git a/libgo/go/archive/tar/reader_test.go b/libgo/go/archive/tar/reader_test.go new file mode 100644 index 000000000..aa4c797fb --- /dev/null +++ b/libgo/go/archive/tar/reader_test.go @@ -0,0 +1,274 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tar + +import ( + "bytes" + "crypto/md5" + "fmt" + "io" + "os" + "reflect" + "testing" +) + +type untarTest struct { + file string + headers []*Header + cksums []string +} + +var gnuTarTest = &untarTest{ + file: "testdata/gnu.tar", + headers: []*Header{ + &Header{ + Name: "small.txt", + Mode: 0640, + Uid: 73025, + Gid: 5000, + Size: 5, + Mtime: 1244428340, + Typeflag: '0', + Uname: "dsymonds", + Gname: "eng", + }, + &Header{ + Name: "small2.txt", + Mode: 0640, + Uid: 73025, + Gid: 5000, + Size: 11, + Mtime: 1244436044, + Typeflag: '0', + Uname: "dsymonds", + Gname: "eng", + }, + }, + cksums: []string{ + "e38b27eaccb4391bdec553a7f3ae6b2f", + "c65bd2e50a56a2138bf1716f2fd56fe9", + }, +} + +var untarTests = []*untarTest{ + gnuTarTest, + &untarTest{ + file: "testdata/star.tar", + headers: []*Header{ + &Header{ + Name: "small.txt", + Mode: 0640, + Uid: 73025, + Gid: 5000, + Size: 5, + Mtime: 1244592783, + Typeflag: '0', + Uname: "dsymonds", + Gname: "eng", + Atime: 1244592783, + Ctime: 1244592783, + }, + &Header{ + Name: "small2.txt", + Mode: 0640, + Uid: 73025, + Gid: 5000, + Size: 11, + Mtime: 1244592783, + Typeflag: '0', + Uname: "dsymonds", + Gname: "eng", + Atime: 1244592783, + Ctime: 1244592783, + }, + }, + }, + &untarTest{ + file: "testdata/v7.tar", + headers: []*Header{ + &Header{ + Name: "small.txt", + Mode: 0444, + Uid: 73025, + Gid: 5000, + Size: 5, + Mtime: 1244593104, + Typeflag: '\x00', + }, + &Header{ + Name: "small2.txt", + Mode: 0444, + Uid: 73025, + Gid: 5000, + Size: 11, + Mtime: 1244593104, + Typeflag: '\x00', + }, + }, + }, +} + +func TestReader(t *testing.T) { +testLoop: + for i, test := range untarTests { + f, err := os.Open(test.file, os.O_RDONLY, 0444) + if err != nil { + t.Errorf("test %d: Unexpected error: %v", i, err) + continue + } + tr := NewReader(f) + for j, header := range test.headers { + hdr, err := tr.Next() + if err != nil || hdr == nil { + t.Errorf("test %d, entry %d: Didn't get entry: %v", i, j, err) + f.Close() + continue testLoop + } + if !reflect.DeepEqual(hdr, header) { + t.Errorf("test %d, entry %d: Incorrect header:\nhave %+v\nwant %+v", + i, j, *hdr, *header) + } + } + hdr, err := tr.Next() + if err == os.EOF { + break + } + if hdr != nil || err != nil { + t.Errorf("test %d: Unexpected entry or error: hdr=%v err=%v", i, hdr, err) + } + f.Close() + } +} + +func TestPartialRead(t *testing.T) { + f, err := os.Open("testdata/gnu.tar", os.O_RDONLY, 0444) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + defer f.Close() + + tr := NewReader(f) + + // Read the first four bytes; Next() should skip the last byte. + hdr, err := tr.Next() + if err != nil || hdr == nil { + t.Fatalf("Didn't get first file: %v", err) + } + buf := make([]byte, 4) + if _, err := io.ReadFull(tr, buf); err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if expected := []byte("Kilt"); !bytes.Equal(buf, expected) { + t.Errorf("Contents = %v, want %v", buf, expected) + } + + // Second file + hdr, err = tr.Next() + if err != nil || hdr == nil { + t.Fatalf("Didn't get second file: %v", err) + } + buf = make([]byte, 6) + if _, err := io.ReadFull(tr, buf); err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if expected := []byte("Google"); !bytes.Equal(buf, expected) { + t.Errorf("Contents = %v, want %v", buf, expected) + } +} + + +func TestIncrementalRead(t *testing.T) { + test := gnuTarTest + f, err := os.Open(test.file, os.O_RDONLY, 0444) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + defer f.Close() + + tr := NewReader(f) + + headers := test.headers + cksums := test.cksums + nread := 0 + + // loop over all files + for ; ; nread++ { + hdr, err := tr.Next() + if hdr == nil || err == os.EOF { + break + } + + // check the header + if !reflect.DeepEqual(hdr, headers[nread]) { + t.Errorf("Incorrect header:\nhave %+v\nwant %+v", + *hdr, headers[nread]) + } + + // read file contents in little chunks EOF, + // checksumming all the way + h := md5.New() + rdbuf := make([]uint8, 8) + for { + nr, err := tr.Read(rdbuf) + if err == os.EOF { + break + } + if err != nil { + t.Errorf("Read: unexpected error %v\n", err) + break + } + h.Write(rdbuf[0:nr]) + } + // verify checksum + have := fmt.Sprintf("%x", h.Sum()) + want := cksums[nread] + if want != have { + t.Errorf("Bad checksum on file %s:\nhave %+v\nwant %+v", hdr.Name, have, want) + } + } + if nread != len(headers) { + t.Errorf("Didn't process all files\nexpected: %d\nprocessed %d\n", len(headers), nread) + } +} + +func TestNonSeekable(t *testing.T) { + test := gnuTarTest + f, err := os.Open(test.file, os.O_RDONLY, 0444) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + defer f.Close() + + // pipe the data in + r, w, err := os.Pipe() + if err != nil { + t.Fatalf("Unexpected error %s", err) + } + go func() { + rdbuf := make([]uint8, 1<<16) + for { + nr, err := f.Read(rdbuf) + w.Write(rdbuf[0:nr]) + if err == os.EOF { + break + } + } + w.Close() + }() + + tr := NewReader(r) + nread := 0 + + for ; ; nread++ { + hdr, err := tr.Next() + if hdr == nil || err == os.EOF { + break + } + } + + if nread != len(test.headers) { + t.Errorf("Didn't process all files\nexpected: %d\nprocessed %d\n", len(test.headers), nread) + } +} diff --git a/libgo/go/archive/tar/testdata/gnu.tar b/libgo/go/archive/tar/testdata/gnu.tar new file mode 100644 index 000000000..fc899dc8d Binary files /dev/null and b/libgo/go/archive/tar/testdata/gnu.tar differ diff --git a/libgo/go/archive/tar/testdata/small.txt b/libgo/go/archive/tar/testdata/small.txt new file mode 100644 index 000000000..b249bfc51 --- /dev/null +++ b/libgo/go/archive/tar/testdata/small.txt @@ -0,0 +1 @@ +Kilts \ No newline at end of file diff --git a/libgo/go/archive/tar/testdata/small2.txt b/libgo/go/archive/tar/testdata/small2.txt new file mode 100644 index 000000000..394ee3ecd --- /dev/null +++ b/libgo/go/archive/tar/testdata/small2.txt @@ -0,0 +1 @@ +Google.com diff --git a/libgo/go/archive/tar/testdata/star.tar b/libgo/go/archive/tar/testdata/star.tar new file mode 100644 index 000000000..59e2d4e60 Binary files /dev/null and b/libgo/go/archive/tar/testdata/star.tar differ diff --git a/libgo/go/archive/tar/testdata/v7.tar b/libgo/go/archive/tar/testdata/v7.tar new file mode 100644 index 000000000..eb65fc941 Binary files /dev/null and b/libgo/go/archive/tar/testdata/v7.tar differ diff --git a/libgo/go/archive/tar/testdata/writer-big.tar b/libgo/go/archive/tar/testdata/writer-big.tar new file mode 100644 index 000000000..753e883ce Binary files /dev/null and b/libgo/go/archive/tar/testdata/writer-big.tar differ diff --git a/libgo/go/archive/tar/testdata/writer.tar b/libgo/go/archive/tar/testdata/writer.tar new file mode 100644 index 000000000..0358f91b9 Binary files /dev/null and b/libgo/go/archive/tar/testdata/writer.tar differ diff --git a/libgo/go/archive/tar/writer.go b/libgo/go/archive/tar/writer.go new file mode 100644 index 000000000..8673bad31 --- /dev/null +++ b/libgo/go/archive/tar/writer.go @@ -0,0 +1,205 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tar + +// TODO(dsymonds): +// - catch more errors (no first header, write after close, etc.) + +import ( + "io" + "os" + "strconv" +) + +var ( + ErrWriteTooLong = os.NewError("write too long") + ErrFieldTooLong = os.NewError("header field too long") + ErrWriteAfterClose = os.NewError("write after close") +) + +// A Writer provides sequential writing of a tar archive in POSIX.1 format. +// A tar archive consists of a sequence of files. +// Call WriteHeader to begin a new file, and then call Write to supply that file's data, +// writing at most hdr.Size bytes in total. +// +// Example: +// tw := tar.NewWriter(w) +// hdr := new(Header) +// hdr.Size = length of data in bytes +// // populate other hdr fields as desired +// if err := tw.WriteHeader(hdr); err != nil { +// // handle error +// } +// io.Copy(tw, data) +// tw.Close() +type Writer struct { + w io.Writer + err os.Error + nb int64 // number of unwritten bytes for current file entry + pad int64 // amount of padding to write after current file entry + closed bool + usedBinary bool // whether the binary numeric field extension was used +} + +// NewWriter creates a new Writer writing to w. +func NewWriter(w io.Writer) *Writer { return &Writer{w: w} } + +// Flush finishes writing the current file (optional). +func (tw *Writer) Flush() os.Error { + n := tw.nb + tw.pad + for n > 0 && tw.err == nil { + nr := n + if nr > blockSize { + nr = blockSize + } + var nw int + nw, tw.err = tw.w.Write(zeroBlock[0:nr]) + n -= int64(nw) + } + tw.nb = 0 + tw.pad = 0 + return tw.err +} + +// Write s into b, terminating it with a NUL if there is room. +func (tw *Writer) cString(b []byte, s string) { + if len(s) > len(b) { + if tw.err == nil { + tw.err = ErrFieldTooLong + } + return + } + copy(b, s) + if len(s) < len(b) { + b[len(s)] = 0 + } +} + +// Encode x as an octal ASCII string and write it into b with leading zeros. +func (tw *Writer) octal(b []byte, x int64) { + s := strconv.Itob64(x, 8) + // leading zeros, but leave room for a NUL. + for len(s)+1 < len(b) { + s = "0" + s + } + tw.cString(b, s) +} + +// Write x into b, either as octal or as binary (GNUtar/star extension). +func (tw *Writer) numeric(b []byte, x int64) { + // Try octal first. + s := strconv.Itob64(x, 8) + if len(s) < len(b) { + tw.octal(b, x) + return + } + // Too big: use binary (big-endian). + tw.usedBinary = true + for i := len(b) - 1; x > 0 && i >= 0; i-- { + b[i] = byte(x) + x >>= 8 + } + b[0] |= 0x80 // highest bit indicates binary format +} + +// WriteHeader writes hdr and prepares to accept the file's contents. +// WriteHeader calls Flush if it is not the first header. +// Calling after a Close will return ErrWriteAfterClose. +func (tw *Writer) WriteHeader(hdr *Header) os.Error { + if tw.closed { + return ErrWriteAfterClose + } + if tw.err == nil { + tw.Flush() + } + if tw.err != nil { + return tw.err + } + + tw.nb = int64(hdr.Size) + tw.pad = -tw.nb & (blockSize - 1) // blockSize is a power of two + + header := make([]byte, blockSize) + s := slicer(header) + + // TODO(dsymonds): handle names longer than 100 chars + copy(s.next(100), []byte(hdr.Name)) + + tw.octal(s.next(8), hdr.Mode) // 100:108 + tw.numeric(s.next(8), int64(hdr.Uid)) // 108:116 + tw.numeric(s.next(8), int64(hdr.Gid)) // 116:124 + tw.numeric(s.next(12), hdr.Size) // 124:136 + tw.numeric(s.next(12), hdr.Mtime) // 136:148 + s.next(8) // chksum (148:156) + s.next(1)[0] = hdr.Typeflag // 156:157 + s.next(100) // linkname (157:257) + copy(s.next(8), []byte("ustar\x0000")) // 257:265 + tw.cString(s.next(32), hdr.Uname) // 265:297 + tw.cString(s.next(32), hdr.Gname) // 297:329 + tw.numeric(s.next(8), hdr.Devmajor) // 329:337 + tw.numeric(s.next(8), hdr.Devminor) // 337:345 + + // Use the GNU magic instead of POSIX magic if we used any GNU extensions. + if tw.usedBinary { + copy(header[257:265], []byte("ustar \x00")) + } + + // The chksum field is terminated by a NUL and a space. + // This is different from the other octal fields. + chksum, _ := checksum(header) + tw.octal(header[148:155], chksum) + header[155] = ' ' + + if tw.err != nil { + // problem with header; probably integer too big for a field. + return tw.err + } + + _, tw.err = tw.w.Write(header) + + return tw.err +} + +// Write writes to the current entry in the tar archive. +// Write returns the error ErrWriteTooLong if more than +// hdr.Size bytes are written after WriteHeader. +func (tw *Writer) Write(b []byte) (n int, err os.Error) { + if tw.closed { + err = ErrWriteTooLong + return + } + overwrite := false + if int64(len(b)) > tw.nb { + b = b[0:tw.nb] + overwrite = true + } + n, err = tw.w.Write(b) + tw.nb -= int64(n) + if err == nil && overwrite { + err = ErrWriteTooLong + return + } + tw.err = err + return +} + +// Close closes the tar archive, flushing any unwritten +// data to the underlying writer. +func (tw *Writer) Close() os.Error { + if tw.err != nil || tw.closed { + return tw.err + } + tw.Flush() + tw.closed = true + + // trailer: two zero blocks + for i := 0; i < 2; i++ { + _, tw.err = tw.w.Write(zeroBlock) + if tw.err != nil { + break + } + } + return tw.err +} diff --git a/libgo/go/archive/tar/writer_test.go b/libgo/go/archive/tar/writer_test.go new file mode 100644 index 000000000..48b891140 --- /dev/null +++ b/libgo/go/archive/tar/writer_test.go @@ -0,0 +1,154 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tar + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "testing" + "testing/iotest" +) + +type writerTestEntry struct { + header *Header + contents string +} + +type writerTest struct { + file string // filename of expected output + entries []*writerTestEntry +} + +var writerTests = []*writerTest{ + &writerTest{ + file: "testdata/writer.tar", + entries: []*writerTestEntry{ + &writerTestEntry{ + header: &Header{ + Name: "small.txt", + Mode: 0640, + Uid: 73025, + Gid: 5000, + Size: 5, + Mtime: 1246508266, + Typeflag: '0', + Uname: "dsymonds", + Gname: "eng", + }, + contents: "Kilts", + }, + &writerTestEntry{ + header: &Header{ + Name: "small2.txt", + Mode: 0640, + Uid: 73025, + Gid: 5000, + Size: 11, + Mtime: 1245217492, + Typeflag: '0', + Uname: "dsymonds", + Gname: "eng", + }, + contents: "Google.com\n", + }, + }, + }, + // The truncated test file was produced using these commands: + // dd if=/dev/zero bs=1048576 count=16384 > /tmp/16gig.txt + // tar -b 1 -c -f- /tmp/16gig.txt | dd bs=512 count=8 > writer-big.tar + &writerTest{ + file: "testdata/writer-big.tar", + entries: []*writerTestEntry{ + &writerTestEntry{ + header: &Header{ + Name: "tmp/16gig.txt", + Mode: 0640, + Uid: 73025, + Gid: 5000, + Size: 16 << 30, + Mtime: 1254699560, + Typeflag: '0', + Uname: "dsymonds", + Gname: "eng", + }, + // no contents + }, + }, + }, +} + +// Render byte array in a two-character hexadecimal string, spaced for easy visual inspection. +func bytestr(offset int, b []byte) string { + const rowLen = 32 + s := fmt.Sprintf("%04x ", offset) + for _, ch := range b { + switch { + case '0' <= ch && ch <= '9', 'A' <= ch && ch <= 'Z', 'a' <= ch && ch <= 'z': + s += fmt.Sprintf(" %c", ch) + default: + s += fmt.Sprintf(" %02x", ch) + } + } + return s +} + +// Render a pseudo-diff between two blocks of bytes. +func bytediff(a []byte, b []byte) string { + const rowLen = 32 + s := fmt.Sprintf("(%d bytes vs. %d bytes)\n", len(a), len(b)) + for offset := 0; len(a)+len(b) > 0; offset += rowLen { + na, nb := rowLen, rowLen + if na > len(a) { + na = len(a) + } + if nb > len(b) { + nb = len(b) + } + sa := bytestr(offset, a[0:na]) + sb := bytestr(offset, b[0:nb]) + if sa != sb { + s += fmt.Sprintf("-%v\n+%v\n", sa, sb) + } + a = a[na:] + b = b[nb:] + } + return s +} + +func TestWriter(t *testing.T) { +testLoop: + for i, test := range writerTests { + expected, err := ioutil.ReadFile(test.file) + if err != nil { + t.Errorf("test %d: Unexpected error: %v", i, err) + continue + } + + buf := new(bytes.Buffer) + tw := NewWriter(iotest.TruncateWriter(buf, 4<<10)) // only catch the first 4 KB + for j, entry := range test.entries { + if err := tw.WriteHeader(entry.header); err != nil { + t.Errorf("test %d, entry %d: Failed writing header: %v", i, j, err) + continue testLoop + } + if _, err := io.WriteString(tw, entry.contents); err != nil { + t.Errorf("test %d, entry %d: Failed writing contents: %v", i, j, err) + continue testLoop + } + } + if err := tw.Close(); err != nil { + t.Errorf("test %d: Failed closing archive: %v", i, err) + continue testLoop + } + + actual := buf.Bytes() + if !bytes.Equal(expected, actual) { + t.Errorf("test %d: Incorrect result: (-=expected, +=actual)\n%v", + i, bytediff(expected, actual)) + } + } +} diff --git a/libgo/go/archive/zip/reader.go b/libgo/go/archive/zip/reader.go new file mode 100644 index 000000000..579ba1602 --- /dev/null +++ b/libgo/go/archive/zip/reader.go @@ -0,0 +1,278 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +The zip package provides support for reading ZIP archives. + +See: http://www.pkware.com/documents/casestudies/APPNOTE.TXT + +This package does not support ZIP64 or disk spanning. +*/ +package zip + +import ( + "bufio" + "bytes" + "compress/flate" + "hash" + "hash/crc32" + "encoding/binary" + "io" + "os" +) + +var ( + FormatError = os.NewError("not a valid zip file") + UnsupportedMethod = os.NewError("unsupported compression algorithm") + ChecksumError = os.NewError("checksum error") +) + +type Reader struct { + r io.ReaderAt + File []*File + Comment string +} + +type File struct { + FileHeader + zipr io.ReaderAt + zipsize int64 + headerOffset uint32 + bodyOffset int64 +} + +// OpenReader will open the Zip file specified by name and return a Reader. +func OpenReader(name string) (*Reader, os.Error) { + f, err := os.Open(name, os.O_RDONLY, 0644) + if err != nil { + return nil, err + } + fi, err := f.Stat() + if err != nil { + return nil, err + } + return NewReader(f, fi.Size) +} + +// NewReader returns a new Reader reading from r, which is assumed to +// have the given size in bytes. +func NewReader(r io.ReaderAt, size int64) (*Reader, os.Error) { + end, err := readDirectoryEnd(r, size) + if err != nil { + return nil, err + } + z := &Reader{ + r: r, + File: make([]*File, end.directoryRecords), + Comment: end.comment, + } + rs := io.NewSectionReader(r, 0, size) + if _, err = rs.Seek(int64(end.directoryOffset), 0); err != nil { + return nil, err + } + buf := bufio.NewReader(rs) + for i := range z.File { + z.File[i] = &File{zipr: r, zipsize: size} + if err := readDirectoryHeader(z.File[i], buf); err != nil { + return nil, err + } + } + return z, nil +} + +// Open returns a ReadCloser that provides access to the File's contents. +func (f *File) Open() (rc io.ReadCloser, err os.Error) { + off := int64(f.headerOffset) + if f.bodyOffset == 0 { + r := io.NewSectionReader(f.zipr, off, f.zipsize-off) + if err = readFileHeader(f, r); err != nil { + return + } + if f.bodyOffset, err = r.Seek(0, 1); err != nil { + return + } + } + r := io.NewSectionReader(f.zipr, off+f.bodyOffset, int64(f.CompressedSize)) + switch f.Method { + case 0: // store (no compression) + rc = nopCloser{r} + case 8: // DEFLATE + rc = flate.NewReader(r) + default: + err = UnsupportedMethod + } + if rc != nil { + rc = &checksumReader{rc, crc32.NewIEEE(), f.CRC32} + } + return +} + +type checksumReader struct { + rc io.ReadCloser + hash hash.Hash32 + sum uint32 +} + +func (r *checksumReader) Read(b []byte) (n int, err os.Error) { + n, err = r.rc.Read(b) + r.hash.Write(b[:n]) + if err != os.EOF { + return + } + if r.hash.Sum32() != r.sum { + err = ChecksumError + } + return +} + +func (r *checksumReader) Close() os.Error { return r.rc.Close() } + +type nopCloser struct { + io.Reader +} + +func (f nopCloser) Close() os.Error { return nil } + +func readFileHeader(f *File, r io.Reader) (err os.Error) { + defer func() { + if rerr, ok := recover().(os.Error); ok { + err = rerr + } + }() + var ( + signature uint32 + filenameLength uint16 + extraLength uint16 + ) + read(r, &signature) + if signature != fileHeaderSignature { + return FormatError + } + read(r, &f.ReaderVersion) + read(r, &f.Flags) + read(r, &f.Method) + read(r, &f.ModifiedTime) + read(r, &f.ModifiedDate) + read(r, &f.CRC32) + read(r, &f.CompressedSize) + read(r, &f.UncompressedSize) + read(r, &filenameLength) + read(r, &extraLength) + f.Name = string(readByteSlice(r, filenameLength)) + f.Extra = readByteSlice(r, extraLength) + return +} + +func readDirectoryHeader(f *File, r io.Reader) (err os.Error) { + defer func() { + if rerr, ok := recover().(os.Error); ok { + err = rerr + } + }() + var ( + signature uint32 + filenameLength uint16 + extraLength uint16 + commentLength uint16 + startDiskNumber uint16 // unused + internalAttributes uint16 // unused + externalAttributes uint32 // unused + ) + read(r, &signature) + if signature != directoryHeaderSignature { + return FormatError + } + read(r, &f.CreatorVersion) + read(r, &f.ReaderVersion) + read(r, &f.Flags) + read(r, &f.Method) + read(r, &f.ModifiedTime) + read(r, &f.ModifiedDate) + read(r, &f.CRC32) + read(r, &f.CompressedSize) + read(r, &f.UncompressedSize) + read(r, &filenameLength) + read(r, &extraLength) + read(r, &commentLength) + read(r, &startDiskNumber) + read(r, &internalAttributes) + read(r, &externalAttributes) + read(r, &f.headerOffset) + f.Name = string(readByteSlice(r, filenameLength)) + f.Extra = readByteSlice(r, extraLength) + f.Comment = string(readByteSlice(r, commentLength)) + return +} + +func readDirectoryEnd(r io.ReaderAt, size int64) (d *directoryEnd, err os.Error) { + // look for directoryEndSignature in the last 1k, then in the last 65k + var b []byte + for i, bLen := range []int64{1024, 65 * 1024} { + if bLen > size { + bLen = size + } + b = make([]byte, int(bLen)) + if _, err := r.ReadAt(b, size-bLen); err != nil && err != os.EOF { + return nil, err + } + if p := findSignatureInBlock(b); p >= 0 { + b = b[p:] + break + } + if i == 1 || bLen == size { + return nil, FormatError + } + } + + // read header into struct + defer func() { + if rerr, ok := recover().(os.Error); ok { + err = rerr + d = nil + } + }() + br := bytes.NewBuffer(b[4:]) // skip over signature + d = new(directoryEnd) + read(br, &d.diskNbr) + read(br, &d.dirDiskNbr) + read(br, &d.dirRecordsThisDisk) + read(br, &d.directoryRecords) + read(br, &d.directorySize) + read(br, &d.directoryOffset) + read(br, &d.commentLen) + d.comment = string(readByteSlice(br, d.commentLen)) + return d, nil +} + +func findSignatureInBlock(b []byte) int { + const minSize = 4 + 2 + 2 + 2 + 2 + 4 + 4 + 2 // fixed part of header + for i := len(b) - minSize; i >= 0; i-- { + // defined from directoryEndSignature in struct.go + if b[i] == 'P' && b[i+1] == 'K' && b[i+2] == 0x05 && b[i+3] == 0x06 { + // n is length of comment + n := int(b[i+minSize-2]) | int(b[i+minSize-1])<<8 + if n+minSize+i == len(b) { + return i + } + } + } + return -1 +} + +func read(r io.Reader, data interface{}) { + if err := binary.Read(r, binary.LittleEndian, data); err != nil { + panic(err) + } +} + +func readByteSlice(r io.Reader, l uint16) []byte { + b := make([]byte, l) + if l == 0 { + return b + } + if _, err := io.ReadFull(r, b); err != nil { + panic(err) + } + return b +} diff --git a/libgo/go/archive/zip/reader_test.go b/libgo/go/archive/zip/reader_test.go new file mode 100644 index 000000000..3c24f1467 --- /dev/null +++ b/libgo/go/archive/zip/reader_test.go @@ -0,0 +1,180 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package zip + +import ( + "bytes" + "encoding/binary" + "io" + "io/ioutil" + "os" + "testing" +) + +type ZipTest struct { + Name string + Comment string + File []ZipTestFile + Error os.Error // the error that Opening this file should return +} + +type ZipTestFile struct { + Name string + Content []byte // if blank, will attempt to compare against File + File string // name of file to compare to (relative to testdata/) +} + +var tests = []ZipTest{ + { + Name: "test.zip", + Comment: "This is a zipfile comment.", + File: []ZipTestFile{ + { + Name: "test.txt", + Content: []byte("This is a test text file.\n"), + }, + { + Name: "gophercolor16x16.png", + File: "gophercolor16x16.png", + }, + }, + }, + { + Name: "r.zip", + File: []ZipTestFile{ + { + Name: "r/r.zip", + File: "r.zip", + }, + }, + }, + {Name: "readme.zip"}, + {Name: "readme.notzip", Error: FormatError}, +} + +func TestReader(t *testing.T) { + for _, zt := range tests { + readTestZip(t, zt) + } +} + +func readTestZip(t *testing.T, zt ZipTest) { + z, err := OpenReader("testdata/" + zt.Name) + if err != zt.Error { + t.Errorf("error=%v, want %v", err, zt.Error) + return + } + + // bail here if no Files expected to be tested + // (there may actually be files in the zip, but we don't care) + if zt.File == nil { + return + } + + if z.Comment != zt.Comment { + t.Errorf("%s: comment=%q, want %q", zt.Name, z.Comment, zt.Comment) + } + if len(z.File) != len(zt.File) { + t.Errorf("%s: file count=%d, want %d", zt.Name, len(z.File), len(zt.File)) + } + + // test read of each file + for i, ft := range zt.File { + readTestFile(t, ft, z.File[i]) + } + + // test simultaneous reads + n := 0 + done := make(chan bool) + for i := 0; i < 5; i++ { + for j, ft := range zt.File { + go func() { + readTestFile(t, ft, z.File[j]) + done <- true + }() + n++ + } + } + for ; n > 0; n-- { + <-done + } + + // test invalid checksum + z.File[0].CRC32++ // invalidate + r, err := z.File[0].Open() + if err != nil { + t.Error(err) + return + } + var b bytes.Buffer + _, err = io.Copy(&b, r) + if err != ChecksumError { + t.Errorf("%s: copy error=%v, want %v", z.File[0].Name, err, ChecksumError) + } +} + +func readTestFile(t *testing.T, ft ZipTestFile, f *File) { + if f.Name != ft.Name { + t.Errorf("name=%q, want %q", f.Name, ft.Name) + } + var b bytes.Buffer + r, err := f.Open() + if err != nil { + t.Error(err) + return + } + _, err = io.Copy(&b, r) + if err != nil { + t.Error(err) + return + } + r.Close() + var c []byte + if len(ft.Content) != 0 { + c = ft.Content + } else if c, err = ioutil.ReadFile("testdata/" + ft.File); err != nil { + t.Error(err) + return + } + if b.Len() != len(c) { + t.Errorf("%s: len=%d, want %d", f.Name, b.Len(), len(c)) + return + } + for i, b := range b.Bytes() { + if b != c[i] { + t.Errorf("%s: content[%d]=%q want %q", f.Name, i, b, c[i]) + return + } + } +} + +func TestInvalidFiles(t *testing.T) { + const size = 1024 * 70 // 70kb + b := make([]byte, size) + + // zeroes + _, err := NewReader(sliceReaderAt(b), size) + if err != FormatError { + t.Errorf("zeroes: error=%v, want %v", err, FormatError) + } + + // repeated directoryEndSignatures + sig := make([]byte, 4) + binary.LittleEndian.PutUint32(sig, directoryEndSignature) + for i := 0; i < size-4; i += 4 { + copy(b[i:i+4], sig) + } + _, err = NewReader(sliceReaderAt(b), size) + if err != FormatError { + t.Errorf("sigs: error=%v, want %v", err, FormatError) + } +} + +type sliceReaderAt []byte + +func (r sliceReaderAt) ReadAt(b []byte, off int64) (int, os.Error) { + copy(b, r[int(off):int(off)+len(b)]) + return len(b), nil +} diff --git a/libgo/go/archive/zip/struct.go b/libgo/go/archive/zip/struct.go new file mode 100644 index 000000000..8a8c727d4 --- /dev/null +++ b/libgo/go/archive/zip/struct.go @@ -0,0 +1,33 @@ +package zip + +const ( + fileHeaderSignature = 0x04034b50 + directoryHeaderSignature = 0x02014b50 + directoryEndSignature = 0x06054b50 +) + +type FileHeader struct { + Name string + CreatorVersion uint16 + ReaderVersion uint16 + Flags uint16 + Method uint16 + ModifiedTime uint16 + ModifiedDate uint16 + CRC32 uint32 + CompressedSize uint32 + UncompressedSize uint32 + Extra []byte + Comment string +} + +type directoryEnd struct { + diskNbr uint16 // unused + dirDiskNbr uint16 // unused + dirRecordsThisDisk uint16 // unused + directoryRecords uint16 + directorySize uint32 + directoryOffset uint32 // relative to file + commentLen uint16 + comment string +} diff --git a/libgo/go/archive/zip/testdata/gophercolor16x16.png b/libgo/go/archive/zip/testdata/gophercolor16x16.png new file mode 100644 index 000000000..48854ff3b Binary files /dev/null and b/libgo/go/archive/zip/testdata/gophercolor16x16.png differ diff --git a/libgo/go/archive/zip/testdata/r.zip b/libgo/go/archive/zip/testdata/r.zip new file mode 100644 index 000000000..ea0fa2ffc Binary files /dev/null and b/libgo/go/archive/zip/testdata/r.zip differ diff --git a/libgo/go/archive/zip/testdata/readme.notzip b/libgo/go/archive/zip/testdata/readme.notzip new file mode 100644 index 000000000..06668c4c1 Binary files /dev/null and b/libgo/go/archive/zip/testdata/readme.notzip differ diff --git a/libgo/go/archive/zip/testdata/readme.zip b/libgo/go/archive/zip/testdata/readme.zip new file mode 100644 index 000000000..db3bb900e Binary files /dev/null and b/libgo/go/archive/zip/testdata/readme.zip differ diff --git a/libgo/go/archive/zip/testdata/test.zip b/libgo/go/archive/zip/testdata/test.zip new file mode 100644 index 000000000..03890c05d Binary files /dev/null and b/libgo/go/archive/zip/testdata/test.zip differ diff --git a/libgo/go/asn1/asn1.go b/libgo/go/asn1/asn1.go new file mode 100644 index 000000000..d06b1d4d7 --- /dev/null +++ b/libgo/go/asn1/asn1.go @@ -0,0 +1,785 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The asn1 package implements parsing of DER-encoded ASN.1 data structures, +// as defined in ITU-T Rec X.690. +// +// See also ``A Layman's Guide to a Subset of ASN.1, BER, and DER,'' +// http://luca.ntop.org/Teaching/Appunti/asn1.html. +package asn1 + +// ASN.1 is a syntax for specifying abstract objects and BER, DER, PER, XER etc +// are different encoding formats for those objects. Here, we'll be dealing +// with DER, the Distinguished Encoding Rules. DER is used in X.509 because +// it's fast to parse and, unlike BER, has a unique encoding for every object. +// When calculating hashes over objects, it's important that the resulting +// bytes be the same at both ends and DER removes this margin of error. +// +// ASN.1 is very complex and this package doesn't attempt to implement +// everything by any means. + +import ( + "fmt" + "os" + "reflect" + "time" +) + +// A StructuralError suggests that the ASN.1 data is valid, but the Go type +// which is receiving it doesn't match. +type StructuralError struct { + Msg string +} + +func (e StructuralError) String() string { return "ASN.1 structure error: " + e.Msg } + +// A SyntaxError suggests that the ASN.1 data is invalid. +type SyntaxError struct { + Msg string +} + +func (e SyntaxError) String() string { return "ASN.1 syntax error: " + e.Msg } + +// We start by dealing with each of the primitive types in turn. + +// BOOLEAN + +func parseBool(bytes []byte) (ret bool, err os.Error) { + if len(bytes) != 1 { + err = SyntaxError{"invalid boolean"} + return + } + + return bytes[0] != 0, nil +} + +// INTEGER + +// parseInt64 treats the given bytes as a big-endian, signed integer and +// returns the result. +func parseInt64(bytes []byte) (ret int64, err os.Error) { + if len(bytes) > 8 { + // We'll overflow an int64 in this case. + err = StructuralError{"integer too large"} + return + } + for bytesRead := 0; bytesRead < len(bytes); bytesRead++ { + ret <<= 8 + ret |= int64(bytes[bytesRead]) + } + + // Shift up and down in order to sign extend the result. + ret <<= 64 - uint8(len(bytes))*8 + ret >>= 64 - uint8(len(bytes))*8 + return +} + +// parseInt treats the given bytes as a big-endian, signed integer and returns +// the result. +func parseInt(bytes []byte) (int, os.Error) { + ret64, err := parseInt64(bytes) + if err != nil { + return 0, err + } + if ret64 != int64(int(ret64)) { + return 0, StructuralError{"integer too large"} + } + return int(ret64), nil +} + +// BIT STRING + +// BitString is the structure to use when you want an ASN.1 BIT STRING type. A +// bit string is padded up to the nearest byte in memory and the number of +// valid bits is recorded. Padding bits will be zero. +type BitString struct { + Bytes []byte // bits packed into bytes. + BitLength int // length in bits. +} + +// At returns the bit at the given index. If the index is out of range it +// returns false. +func (b BitString) At(i int) int { + if i < 0 || i >= b.BitLength { + return 0 + } + x := i / 8 + y := 7 - uint(i%8) + return int(b.Bytes[x]>>y) & 1 +} + +// RightAlign returns a slice where the padding bits are at the beginning. The +// slice may share memory with the BitString. +func (b BitString) RightAlign() []byte { + shift := uint(8 - (b.BitLength % 8)) + if shift == 8 || len(b.Bytes) == 0 { + return b.Bytes + } + + a := make([]byte, len(b.Bytes)) + a[0] = b.Bytes[0] >> shift + for i := 1; i < len(b.Bytes); i++ { + a[i] = b.Bytes[i-1] << (8 - shift) + a[i] |= b.Bytes[i] >> shift + } + + return a +} + +// parseBitString parses an ASN.1 bit string from the given byte array and returns it. +func parseBitString(bytes []byte) (ret BitString, err os.Error) { + if len(bytes) == 0 { + err = SyntaxError{"zero length BIT STRING"} + return + } + paddingBits := int(bytes[0]) + if paddingBits > 7 || + len(bytes) == 1 && paddingBits > 0 || + bytes[len(bytes)-1]&((1< 4 { + err = StructuralError{"base 128 integer too large"} + return + } + ret <<= 7 + b := bytes[offset] + ret |= int(b & 0x7f) + offset++ + if b&0x80 == 0 { + return + } + } + err = SyntaxError{"truncated base 128 integer"} + return +} + +// UTCTime + +func parseUTCTime(bytes []byte) (ret *time.Time, err os.Error) { + s := string(bytes) + ret, err = time.Parse("0601021504Z0700", s) + if err == nil { + return + } + ret, err = time.Parse("060102150405Z0700", s) + return +} + +// parseGeneralizedTime parses the GeneralizedTime from the given byte array +// and returns the resulting time. +func parseGeneralizedTime(bytes []byte) (ret *time.Time, err os.Error) { + return time.Parse("20060102150405Z0700", string(bytes)) +} + +// PrintableString + +// parsePrintableString parses a ASN.1 PrintableString from the given byte +// array and returns it. +func parsePrintableString(bytes []byte) (ret string, err os.Error) { + for _, b := range bytes { + if !isPrintable(b) { + err = SyntaxError{"PrintableString contains invalid character"} + return + } + } + ret = string(bytes) + return +} + +// isPrintable returns true iff the given b is in the ASN.1 PrintableString set. +func isPrintable(b byte) bool { + return 'a' <= b && b <= 'z' || + 'A' <= b && b <= 'Z' || + '0' <= b && b <= '9' || + '\'' <= b && b <= ')' || + '+' <= b && b <= '/' || + b == ' ' || + b == ':' || + b == '=' || + b == '?' || + // This is techincally not allowed in a PrintableString. + // However, x509 certificates with wildcard strings don't + // always use the correct string type so we permit it. + b == '*' +} + +// IA5String + +// parseIA5String parses a ASN.1 IA5String (ASCII string) from the given +// byte array and returns it. +func parseIA5String(bytes []byte) (ret string, err os.Error) { + for _, b := range bytes { + if b >= 0x80 { + err = SyntaxError{"IA5String contains invalid character"} + return + } + } + ret = string(bytes) + return +} + +// T61String + +// parseT61String parses a ASN.1 T61String (8-bit clean string) from the given +// byte array and returns it. +func parseT61String(bytes []byte) (ret string, err os.Error) { + return string(bytes), nil +} + +// A RawValue represents an undecoded ASN.1 object. +type RawValue struct { + Class, Tag int + IsCompound bool + Bytes []byte + FullBytes []byte // includes the tag and length +} + +// RawContent is used to signal that the undecoded, DER data needs to be +// preserved for a struct. To use it, the first field of the struct must have +// this type. It's an error for any of the other fields to have this type. +type RawContent []byte + +// Tagging + +// parseTagAndLength parses an ASN.1 tag and length pair from the given offset +// into a byte array. It returns the parsed data and the new offset. SET and +// SET OF (tag 17) are mapped to SEQUENCE and SEQUENCE OF (tag 16) since we +// don't distinguish between ordered and unordered objects in this code. +func parseTagAndLength(bytes []byte, initOffset int) (ret tagAndLength, offset int, err os.Error) { + offset = initOffset + b := bytes[offset] + offset++ + ret.class = int(b >> 6) + ret.isCompound = b&0x20 == 0x20 + ret.tag = int(b & 0x1f) + + // If the bottom five bits are set, then the tag number is actually base 128 + // encoded afterwards + if ret.tag == 0x1f { + ret.tag, offset, err = parseBase128Int(bytes, offset) + if err != nil { + return + } + } + if offset >= len(bytes) { + err = SyntaxError{"truncated tag or length"} + return + } + b = bytes[offset] + offset++ + if b&0x80 == 0 { + // The length is encoded in the bottom 7 bits. + ret.length = int(b & 0x7f) + } else { + // Bottom 7 bits give the number of length bytes to follow. + numBytes := int(b & 0x7f) + // We risk overflowing a signed 32-bit number if we accept more than 3 bytes. + if numBytes > 3 { + err = StructuralError{"length too large"} + return + } + if numBytes == 0 { + err = SyntaxError{"indefinite length found (not DER)"} + return + } + ret.length = 0 + for i := 0; i < numBytes; i++ { + if offset >= len(bytes) { + err = SyntaxError{"truncated tag or length"} + return + } + b = bytes[offset] + offset++ + ret.length <<= 8 + ret.length |= int(b) + } + } + + return +} + +// parseSequenceOf is used for SEQUENCE OF and SET OF values. It tries to parse +// a number of ASN.1 values from the given byte array and returns them as a +// slice of Go values of the given type. +func parseSequenceOf(bytes []byte, sliceType *reflect.SliceType, elemType reflect.Type) (ret *reflect.SliceValue, err os.Error) { + expectedTag, compoundType, ok := getUniversalType(elemType) + if !ok { + err = StructuralError{"unknown Go type for slice"} + return + } + + // First we iterate over the input and count the number of elements, + // checking that the types are correct in each case. + numElements := 0 + for offset := 0; offset < len(bytes); { + var t tagAndLength + t, offset, err = parseTagAndLength(bytes, offset) + if err != nil { + return + } + if t.class != classUniversal || t.isCompound != compoundType || t.tag != expectedTag { + err = StructuralError{"sequence tag mismatch"} + return + } + if invalidLength(offset, t.length, len(bytes)) { + err = SyntaxError{"truncated sequence"} + return + } + offset += t.length + numElements++ + } + ret = reflect.MakeSlice(sliceType, numElements, numElements) + params := fieldParameters{} + offset := 0 + for i := 0; i < numElements; i++ { + offset, err = parseField(ret.Elem(i), bytes, offset, params) + if err != nil { + return + } + } + return +} + +var ( + bitStringType = reflect.Typeof(BitString{}) + objectIdentifierType = reflect.Typeof(ObjectIdentifier{}) + enumeratedType = reflect.Typeof(Enumerated(0)) + flagType = reflect.Typeof(Flag(false)) + timeType = reflect.Typeof(&time.Time{}) + rawValueType = reflect.Typeof(RawValue{}) + rawContentsType = reflect.Typeof(RawContent(nil)) +) + +// invalidLength returns true iff offset + length > sliceLength, or if the +// addition would overflow. +func invalidLength(offset, length, sliceLength int) bool { + return offset+length < offset || offset+length > sliceLength +} + +// parseField is the main parsing function. Given a byte array and an offset +// into the array, it will try to parse a suitable ASN.1 value out and store it +// in the given Value. +func parseField(v reflect.Value, bytes []byte, initOffset int, params fieldParameters) (offset int, err os.Error) { + offset = initOffset + fieldType := v.Type() + + // If we have run out of data, it may be that there are optional elements at the end. + if offset == len(bytes) { + if !setDefaultValue(v, params) { + err = SyntaxError{"sequence truncated"} + } + return + } + + // Deal with raw values. + if fieldType == rawValueType { + var t tagAndLength + t, offset, err = parseTagAndLength(bytes, offset) + if err != nil { + return + } + if invalidLength(offset, t.length, len(bytes)) { + err = SyntaxError{"data truncated"} + return + } + result := RawValue{t.class, t.tag, t.isCompound, bytes[offset : offset+t.length], bytes[initOffset : offset+t.length]} + offset += t.length + v.(*reflect.StructValue).Set(reflect.NewValue(result).(*reflect.StructValue)) + return + } + + // Deal with the ANY type. + if ifaceType, ok := fieldType.(*reflect.InterfaceType); ok && ifaceType.NumMethod() == 0 { + ifaceValue := v.(*reflect.InterfaceValue) + var t tagAndLength + t, offset, err = parseTagAndLength(bytes, offset) + if err != nil { + return + } + if invalidLength(offset, t.length, len(bytes)) { + err = SyntaxError{"data truncated"} + return + } + var result interface{} + if !t.isCompound && t.class == classUniversal { + innerBytes := bytes[offset : offset+t.length] + switch t.tag { + case tagPrintableString: + result, err = parsePrintableString(innerBytes) + case tagIA5String: + result, err = parseIA5String(innerBytes) + case tagT61String: + result, err = parseT61String(innerBytes) + case tagInteger: + result, err = parseInt64(innerBytes) + case tagBitString: + result, err = parseBitString(innerBytes) + case tagOID: + result, err = parseObjectIdentifier(innerBytes) + case tagUTCTime: + result, err = parseUTCTime(innerBytes) + case tagOctetString: + result = innerBytes + default: + // If we don't know how to handle the type, we just leave Value as nil. + } + } + offset += t.length + if err != nil { + return + } + if result != nil { + ifaceValue.Set(reflect.NewValue(result)) + } + return + } + universalTag, compoundType, ok1 := getUniversalType(fieldType) + if !ok1 { + err = StructuralError{fmt.Sprintf("unknown Go type: %v", fieldType)} + return + } + + t, offset, err := parseTagAndLength(bytes, offset) + if err != nil { + return + } + if params.explicit { + if t.class == classContextSpecific && t.tag == *params.tag && (t.length == 0 || t.isCompound) { + if t.length > 0 { + t, offset, err = parseTagAndLength(bytes, offset) + if err != nil { + return + } + } else { + if fieldType != flagType { + err = StructuralError{"Zero length explicit tag was not an asn1.Flag"} + return + } + + flagValue := v.(*reflect.BoolValue) + flagValue.Set(true) + return + } + } else { + // The tags didn't match, it might be an optional element. + ok := setDefaultValue(v, params) + if ok { + offset = initOffset + } else { + err = StructuralError{"explicitly tagged member didn't match"} + } + return + } + } + + // Special case for strings: PrintableString and IA5String both map to + // the Go type string. getUniversalType returns the tag for + // PrintableString when it sees a string so, if we see an IA5String on + // the wire, we change the universal type to match. + if universalTag == tagPrintableString && t.tag == tagIA5String { + universalTag = tagIA5String + } + + // Special case for time: UTCTime and GeneralizedTime both map to the + // Go type time.Time. + if universalTag == tagUTCTime && t.tag == tagGeneralizedTime { + universalTag = tagGeneralizedTime + } + + expectedClass := classUniversal + expectedTag := universalTag + + if !params.explicit && params.tag != nil { + expectedClass = classContextSpecific + expectedTag = *params.tag + } + + // We have unwrapped any explicit tagging at this point. + if t.class != expectedClass || t.tag != expectedTag || t.isCompound != compoundType { + // Tags don't match. Again, it could be an optional element. + ok := setDefaultValue(v, params) + if ok { + offset = initOffset + } else { + err = StructuralError{fmt.Sprintf("tags don't match (%d vs %+v) %+v %s @%d", expectedTag, t, params, fieldType.Name(), offset)} + } + return + } + if invalidLength(offset, t.length, len(bytes)) { + err = SyntaxError{"data truncated"} + return + } + innerBytes := bytes[offset : offset+t.length] + offset += t.length + + // We deal with the structures defined in this package first. + switch fieldType { + case objectIdentifierType: + newSlice, err1 := parseObjectIdentifier(innerBytes) + sliceValue := v.(*reflect.SliceValue) + sliceValue.Set(reflect.MakeSlice(sliceValue.Type().(*reflect.SliceType), len(newSlice), len(newSlice))) + if err1 == nil { + reflect.Copy(sliceValue, reflect.NewValue(newSlice).(reflect.ArrayOrSliceValue)) + } + err = err1 + return + case bitStringType: + structValue := v.(*reflect.StructValue) + bs, err1 := parseBitString(innerBytes) + if err1 == nil { + structValue.Set(reflect.NewValue(bs).(*reflect.StructValue)) + } + err = err1 + return + case timeType: + ptrValue := v.(*reflect.PtrValue) + var time *time.Time + var err1 os.Error + if universalTag == tagUTCTime { + time, err1 = parseUTCTime(innerBytes) + } else { + time, err1 = parseGeneralizedTime(innerBytes) + } + if err1 == nil { + ptrValue.Set(reflect.NewValue(time).(*reflect.PtrValue)) + } + err = err1 + return + case enumeratedType: + parsedInt, err1 := parseInt(innerBytes) + enumValue := v.(*reflect.IntValue) + if err1 == nil { + enumValue.Set(int64(parsedInt)) + } + err = err1 + return + case flagType: + flagValue := v.(*reflect.BoolValue) + flagValue.Set(true) + return + } + switch val := v.(type) { + case *reflect.BoolValue: + parsedBool, err1 := parseBool(innerBytes) + if err1 == nil { + val.Set(parsedBool) + } + err = err1 + return + case *reflect.IntValue: + switch val.Type().Kind() { + case reflect.Int: + parsedInt, err1 := parseInt(innerBytes) + if err1 == nil { + val.Set(int64(parsedInt)) + } + err = err1 + return + case reflect.Int64: + parsedInt, err1 := parseInt64(innerBytes) + if err1 == nil { + val.Set(parsedInt) + } + err = err1 + return + } + case *reflect.StructValue: + structType := fieldType.(*reflect.StructType) + + if structType.NumField() > 0 && + structType.Field(0).Type == rawContentsType { + bytes := bytes[initOffset:offset] + val.Field(0).SetValue(reflect.NewValue(RawContent(bytes))) + } + + innerOffset := 0 + for i := 0; i < structType.NumField(); i++ { + field := structType.Field(i) + if i == 0 && field.Type == rawContentsType { + continue + } + innerOffset, err = parseField(val.Field(i), innerBytes, innerOffset, parseFieldParameters(field.Tag)) + if err != nil { + return + } + } + // We allow extra bytes at the end of the SEQUENCE because + // adding elements to the end has been used in X.509 as the + // version numbers have increased. + return + case *reflect.SliceValue: + sliceType := fieldType.(*reflect.SliceType) + if sliceType.Elem().Kind() == reflect.Uint8 { + val.Set(reflect.MakeSlice(sliceType, len(innerBytes), len(innerBytes))) + reflect.Copy(val, reflect.NewValue(innerBytes).(reflect.ArrayOrSliceValue)) + return + } + newSlice, err1 := parseSequenceOf(innerBytes, sliceType, sliceType.Elem()) + if err1 == nil { + val.Set(newSlice) + } + err = err1 + return + case *reflect.StringValue: + var v string + switch universalTag { + case tagPrintableString: + v, err = parsePrintableString(innerBytes) + case tagIA5String: + v, err = parseIA5String(innerBytes) + case tagT61String: + v, err = parseT61String(innerBytes) + default: + err = SyntaxError{fmt.Sprintf("internal error: unknown string type %d", universalTag)} + } + if err == nil { + val.Set(v) + } + return + } + err = StructuralError{"unknown Go type"} + return +} + +// setDefaultValue is used to install a default value, from a tag string, into +// a Value. It is successful is the field was optional, even if a default value +// wasn't provided or it failed to install it into the Value. +func setDefaultValue(v reflect.Value, params fieldParameters) (ok bool) { + if !params.optional { + return + } + ok = true + if params.defaultValue == nil { + return + } + switch val := v.(type) { + case *reflect.IntValue: + val.Set(*params.defaultValue) + } + return +} + +// Unmarshal parses the DER-encoded ASN.1 data structure b +// and uses the reflect package to fill in an arbitrary value pointed at by val. +// Because Unmarshal uses the reflect package, the structs +// being written to must use upper case field names. +// +// An ASN.1 INTEGER can be written to an int or int64. +// If the encoded value does not fit in the Go type, +// Unmarshal returns a parse error. +// +// An ASN.1 BIT STRING can be written to a BitString. +// +// An ASN.1 OCTET STRING can be written to a []byte. +// +// An ASN.1 OBJECT IDENTIFIER can be written to an +// ObjectIdentifier. +// +// An ASN.1 ENUMERATED can be written to an Enumerated. +// +// An ASN.1 UTCTIME or GENERALIZEDTIME can be written to a *time.Time. +// +// An ASN.1 PrintableString or IA5String can be written to a string. +// +// Any of the above ASN.1 values can be written to an interface{}. +// The value stored in the interface has the corresponding Go type. +// For integers, that type is int64. +// +// An ASN.1 SEQUENCE OF x or SET OF x can be written +// to a slice if an x can be written to the slice's element type. +// +// An ASN.1 SEQUENCE or SET can be written to a struct +// if each of the elements in the sequence can be +// written to the corresponding element in the struct. +// +// The following tags on struct fields have special meaning to Unmarshal: +// +// optional marks the field as ASN.1 OPTIONAL +// [explicit] tag:x specifies the ASN.1 tag number; implies ASN.1 CONTEXT SPECIFIC +// default:x sets the default value for optional integer fields +// +// If the type of the first field of a structure is RawContent then the raw +// ASN1 contents of the struct will be stored in it. +// +// Other ASN.1 types are not supported; if it encounters them, +// Unmarshal returns a parse error. +func Unmarshal(b []byte, val interface{}) (rest []byte, err os.Error) { + v := reflect.NewValue(val).(*reflect.PtrValue).Elem() + offset, err := parseField(v, b, 0, fieldParameters{}) + if err != nil { + return nil, err + } + return b[offset:], nil +} diff --git a/libgo/go/asn1/asn1_test.go b/libgo/go/asn1/asn1_test.go new file mode 100644 index 000000000..34b5f1ecd --- /dev/null +++ b/libgo/go/asn1/asn1_test.go @@ -0,0 +1,634 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package asn1 + +import ( + "bytes" + "reflect" + "testing" + "time" +) + +type int64Test struct { + in []byte + ok bool + out int64 +} + +var int64TestData = []int64Test{ + {[]byte{0x00}, true, 0}, + {[]byte{0x7f}, true, 127}, + {[]byte{0x00, 0x80}, true, 128}, + {[]byte{0x01, 0x00}, true, 256}, + {[]byte{0x80}, true, -128}, + {[]byte{0xff, 0x7f}, true, -129}, + {[]byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, true, -1}, + {[]byte{0xff}, true, -1}, + {[]byte{0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, true, -9223372036854775808}, + {[]byte{0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, false, 0}, +} + +func TestParseInt64(t *testing.T) { + for i, test := range int64TestData { + ret, err := parseInt64(test.in) + if (err == nil) != test.ok { + t.Errorf("#%d: Incorrect error result (did fail? %v, expected: %v)", i, err == nil, test.ok) + } + if test.ok && ret != test.out { + t.Errorf("#%d: Bad result: %v (expected %v)", i, ret, test.out) + } + } +} + +type bitStringTest struct { + in []byte + ok bool + out []byte + bitLength int +} + +var bitStringTestData = []bitStringTest{ + {[]byte{}, false, []byte{}, 0}, + {[]byte{0x00}, true, []byte{}, 0}, + {[]byte{0x07, 0x00}, true, []byte{0x00}, 1}, + {[]byte{0x07, 0x01}, false, []byte{}, 0}, + {[]byte{0x07, 0x40}, false, []byte{}, 0}, + {[]byte{0x08, 0x00}, false, []byte{}, 0}, +} + +func TestBitString(t *testing.T) { + for i, test := range bitStringTestData { + ret, err := parseBitString(test.in) + if (err == nil) != test.ok { + t.Errorf("#%d: Incorrect error result (did fail? %v, expected: %v)", i, err == nil, test.ok) + } + if err == nil { + if test.bitLength != ret.BitLength || bytes.Compare(ret.Bytes, test.out) != 0 { + t.Errorf("#%d: Bad result: %v (expected %v %v)", i, ret, test.out, test.bitLength) + } + } + } +} + +func TestBitStringAt(t *testing.T) { + bs := BitString{[]byte{0x82, 0x40}, 16} + if bs.At(0) != 1 { + t.Error("#1: Failed") + } + if bs.At(1) != 0 { + t.Error("#2: Failed") + } + if bs.At(6) != 1 { + t.Error("#3: Failed") + } + if bs.At(9) != 1 { + t.Error("#4: Failed") + } +} + +type bitStringRightAlignTest struct { + in []byte + inlen int + out []byte +} + +var bitStringRightAlignTests = []bitStringRightAlignTest{ + {[]byte{0x80}, 1, []byte{0x01}}, + {[]byte{0x80, 0x80}, 9, []byte{0x01, 0x01}}, + {[]byte{}, 0, []byte{}}, + {[]byte{0xce}, 8, []byte{0xce}}, + {[]byte{0xce, 0x47}, 16, []byte{0xce, 0x47}}, + {[]byte{0x34, 0x50}, 12, []byte{0x03, 0x45}}, +} + +func TestBitStringRightAlign(t *testing.T) { + for i, test := range bitStringRightAlignTests { + bs := BitString{test.in, test.inlen} + out := bs.RightAlign() + if bytes.Compare(out, test.out) != 0 { + t.Errorf("#%d got: %x want: %x", i, out, test.out) + } + } +} + +type objectIdentifierTest struct { + in []byte + ok bool + out []int +} + +var objectIdentifierTestData = []objectIdentifierTest{ + {[]byte{}, false, []int{}}, + {[]byte{85}, true, []int{2, 5}}, + {[]byte{85, 0x02}, true, []int{2, 5, 2}}, + {[]byte{85, 0x02, 0xc0, 0x00}, true, []int{2, 5, 2, 0x2000}}, + {[]byte{85, 0x02, 0xc0, 0x80, 0x80, 0x80, 0x80}, false, []int{}}, +} + +func TestObjectIdentifier(t *testing.T) { + for i, test := range objectIdentifierTestData { + ret, err := parseObjectIdentifier(test.in) + if (err == nil) != test.ok { + t.Errorf("#%d: Incorrect error result (did fail? %v, expected: %v)", i, err == nil, test.ok) + } + if err == nil { + if !reflect.DeepEqual(test.out, ret) { + t.Errorf("#%d: Bad result: %v (expected %v)", i, ret, test.out) + } + } + } +} + +type timeTest struct { + in string + ok bool + out *time.Time +} + +var utcTestData = []timeTest{ + {"910506164540-0700", true, &time.Time{1991, 05, 06, 16, 45, 40, 0, -7 * 60 * 60, ""}}, + {"910506164540+0730", true, &time.Time{1991, 05, 06, 16, 45, 40, 0, 7*60*60 + 30*60, ""}}, + {"910506234540Z", true, &time.Time{1991, 05, 06, 23, 45, 40, 0, 0, "UTC"}}, + {"9105062345Z", true, &time.Time{1991, 05, 06, 23, 45, 0, 0, 0, "UTC"}}, + {"a10506234540Z", false, nil}, + {"91a506234540Z", false, nil}, + {"9105a6234540Z", false, nil}, + {"910506a34540Z", false, nil}, + {"910506334a40Z", false, nil}, + {"91050633444aZ", false, nil}, + {"910506334461Z", false, nil}, + {"910506334400Za", false, nil}, +} + +func TestUTCTime(t *testing.T) { + for i, test := range utcTestData { + ret, err := parseUTCTime([]byte(test.in)) + if (err == nil) != test.ok { + t.Errorf("#%d: Incorrect error result (did fail? %v, expected: %v)", i, err == nil, test.ok) + } + if err == nil { + if !reflect.DeepEqual(test.out, ret) { + t.Errorf("#%d: Bad result: %v (expected %v)", i, ret, test.out) + } + } + } +} + +var generalizedTimeTestData = []timeTest{ + {"20100102030405Z", true, &time.Time{2010, 01, 02, 03, 04, 05, 0, 0, "UTC"}}, + {"20100102030405", false, nil}, + {"20100102030405+0607", true, &time.Time{2010, 01, 02, 03, 04, 05, 0, 6*60*60 + 7*60, ""}}, + {"20100102030405-0607", true, &time.Time{2010, 01, 02, 03, 04, 05, 0, -6*60*60 - 7*60, ""}}, +} + +func TestGeneralizedTime(t *testing.T) { + for i, test := range generalizedTimeTestData { + ret, err := parseGeneralizedTime([]byte(test.in)) + if (err == nil) != test.ok { + t.Errorf("#%d: Incorrect error result (did fail? %v, expected: %v)", i, err == nil, test.ok) + } + if err == nil { + if !reflect.DeepEqual(test.out, ret) { + t.Errorf("#%d: Bad result: %v (expected %v)", i, ret, test.out) + } + } + } +} + +type tagAndLengthTest struct { + in []byte + ok bool + out tagAndLength +} + +var tagAndLengthData = []tagAndLengthTest{ + {[]byte{0x80, 0x01}, true, tagAndLength{2, 0, 1, false}}, + {[]byte{0xa0, 0x01}, true, tagAndLength{2, 0, 1, true}}, + {[]byte{0x02, 0x00}, true, tagAndLength{0, 2, 0, false}}, + {[]byte{0xfe, 0x00}, true, tagAndLength{3, 30, 0, true}}, + {[]byte{0x1f, 0x01, 0x00}, true, tagAndLength{0, 1, 0, false}}, + {[]byte{0x1f, 0x81, 0x00, 0x00}, true, tagAndLength{0, 128, 0, false}}, + {[]byte{0x1f, 0x81, 0x80, 0x01, 0x00}, true, tagAndLength{0, 0x4001, 0, false}}, + {[]byte{0x00, 0x81, 0x01}, true, tagAndLength{0, 0, 1, false}}, + {[]byte{0x00, 0x82, 0x01, 0x00}, true, tagAndLength{0, 0, 256, false}}, + {[]byte{0x00, 0x83, 0x01, 0x00}, false, tagAndLength{}}, + {[]byte{0x1f, 0x85}, false, tagAndLength{}}, + {[]byte{0x30, 0x80}, false, tagAndLength{}}, +} + +func TestParseTagAndLength(t *testing.T) { + for i, test := range tagAndLengthData { + tagAndLength, _, err := parseTagAndLength(test.in, 0) + if (err == nil) != test.ok { + t.Errorf("#%d: Incorrect error result (did pass? %v, expected: %v)", i, err == nil, test.ok) + } + if err == nil && !reflect.DeepEqual(test.out, tagAndLength) { + t.Errorf("#%d: Bad result: %v (expected %v)", i, tagAndLength, test.out) + } + } +} + +type parseFieldParametersTest struct { + in string + out fieldParameters +} + +func newInt(n int) *int { return &n } + +func newInt64(n int64) *int64 { return &n } + +func newString(s string) *string { return &s } + +func newBool(b bool) *bool { return &b } + +var parseFieldParametersTestData []parseFieldParametersTest = []parseFieldParametersTest{ + {"", fieldParameters{}}, + {"ia5", fieldParameters{stringType: tagIA5String}}, + {"printable", fieldParameters{stringType: tagPrintableString}}, + {"optional", fieldParameters{optional: true}}, + {"explicit", fieldParameters{explicit: true, tag: new(int)}}, + {"optional,explicit", fieldParameters{optional: true, explicit: true, tag: new(int)}}, + {"default:42", fieldParameters{defaultValue: newInt64(42)}}, + {"tag:17", fieldParameters{tag: newInt(17)}}, + {"optional,explicit,default:42,tag:17", fieldParameters{optional: true, explicit: true, defaultValue: newInt64(42), tag: newInt(17)}}, + {"optional,explicit,default:42,tag:17,rubbish1", fieldParameters{true, true, newInt64(42), newInt(17), 0, false}}, + {"set", fieldParameters{set: true}}, +} + +func TestParseFieldParameters(t *testing.T) { + for i, test := range parseFieldParametersTestData { + f := parseFieldParameters(test.in) + if !reflect.DeepEqual(f, test.out) { + t.Errorf("#%d: Bad result: %v (expected %v)", i, f, test.out) + } + } +} + +type unmarshalTest struct { + in []byte + out interface{} +} + +type TestObjectIdentifierStruct struct { + OID ObjectIdentifier +} + +type TestContextSpecificTags struct { + A int "tag:1" +} + +type TestContextSpecificTags2 struct { + A int "explicit,tag:1" + B int +} + +type TestElementsAfterString struct { + S string + A, B int +} + +var unmarshalTestData []unmarshalTest = []unmarshalTest{ + {[]byte{0x02, 0x01, 0x42}, newInt(0x42)}, + {[]byte{0x30, 0x08, 0x06, 0x06, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d}, &TestObjectIdentifierStruct{[]int{1, 2, 840, 113549}}}, + {[]byte{0x03, 0x04, 0x06, 0x6e, 0x5d, 0xc0}, &BitString{[]byte{110, 93, 192}, 18}}, + {[]byte{0x30, 0x09, 0x02, 0x01, 0x01, 0x02, 0x01, 0x02, 0x02, 0x01, 0x03}, &[]int{1, 2, 3}}, + {[]byte{0x02, 0x01, 0x10}, newInt(16)}, + {[]byte{0x13, 0x04, 't', 'e', 's', 't'}, newString("test")}, + {[]byte{0x16, 0x04, 't', 'e', 's', 't'}, newString("test")}, + {[]byte{0x16, 0x04, 't', 'e', 's', 't'}, &RawValue{0, 22, false, []byte("test"), []byte("\x16\x04test")}}, + {[]byte{0x04, 0x04, 1, 2, 3, 4}, &RawValue{0, 4, false, []byte{1, 2, 3, 4}, []byte{4, 4, 1, 2, 3, 4}}}, + {[]byte{0x30, 0x03, 0x81, 0x01, 0x01}, &TestContextSpecificTags{1}}, + {[]byte{0x30, 0x08, 0xa1, 0x03, 0x02, 0x01, 0x01, 0x02, 0x01, 0x02}, &TestContextSpecificTags2{1, 2}}, + {[]byte{0x01, 0x01, 0x00}, newBool(false)}, + {[]byte{0x01, 0x01, 0x01}, newBool(true)}, + {[]byte{0x30, 0x0b, 0x13, 0x03, 0x66, 0x6f, 0x6f, 0x02, 0x01, 0x22, 0x02, 0x01, 0x33}, &TestElementsAfterString{"foo", 0x22, 0x33}}, +} + +func TestUnmarshal(t *testing.T) { + for i, test := range unmarshalTestData { + pv := reflect.MakeZero(reflect.NewValue(test.out).Type()) + zv := reflect.MakeZero(pv.Type().(*reflect.PtrType).Elem()) + pv.(*reflect.PtrValue).PointTo(zv) + val := pv.Interface() + _, err := Unmarshal(test.in, val) + if err != nil { + t.Errorf("Unmarshal failed at index %d %v", i, err) + } + if !reflect.DeepEqual(val, test.out) { + t.Errorf("#%d:\nhave %#v\nwant %#v", i, val, test.out) + } + } +} + +type Certificate struct { + TBSCertificate TBSCertificate + SignatureAlgorithm AlgorithmIdentifier + SignatureValue BitString +} + +type TBSCertificate struct { + Version int "optional,explicit,default:0,tag:0" + SerialNumber RawValue + SignatureAlgorithm AlgorithmIdentifier + Issuer RDNSequence + Validity Validity + Subject RDNSequence + PublicKey PublicKeyInfo +} + +type AlgorithmIdentifier struct { + Algorithm ObjectIdentifier +} + +type RDNSequence []RelativeDistinguishedNameSET + +type RelativeDistinguishedNameSET []AttributeTypeAndValue + +type AttributeTypeAndValue struct { + Type ObjectIdentifier + Value interface{} +} + +type Validity struct { + NotBefore, NotAfter *time.Time +} + +type PublicKeyInfo struct { + Algorithm AlgorithmIdentifier + PublicKey BitString +} + +func TestCertificate(t *testing.T) { + // This is a minimal, self-signed certificate that should parse correctly. + var cert Certificate + if _, err := Unmarshal(derEncodedSelfSignedCertBytes, &cert); err != nil { + t.Errorf("Unmarshal failed: %v", err) + } + if !reflect.DeepEqual(cert, derEncodedSelfSignedCert) { + t.Errorf("Bad result:\ngot: %+v\nwant: %+v", cert, derEncodedSelfSignedCert) + } +} + +func TestCertificateWithNUL(t *testing.T) { + // This is the paypal NUL-hack certificate. It should fail to parse because + // NUL isn't a permitted character in a PrintableString. + + var cert Certificate + if _, err := Unmarshal(derEncodedPaypalNULCertBytes, &cert); err == nil { + t.Error("Unmarshal succeeded, should not have") + } +} + +type rawStructTest struct { + Raw RawContent + A int +} + +func TestRawStructs(t *testing.T) { + var s rawStructTest + input := []byte{0x30, 0x03, 0x02, 0x01, 0x50} + + rest, err := Unmarshal(input, &s) + if len(rest) != 0 { + t.Errorf("incomplete parse: %x", rest) + return + } + if err != nil { + t.Error(err) + return + } + if s.A != 0x50 { + t.Errorf("bad value for A: got %d want %d", s.A, 0x50) + } + if bytes.Compare([]byte(s.Raw), input) != 0 { + t.Errorf("bad value for Raw: got %x want %x", s.Raw, input) + } +} + +var derEncodedSelfSignedCert = Certificate{ + TBSCertificate: TBSCertificate{ + Version: 0, + SerialNumber: RawValue{Class: 0, Tag: 2, IsCompound: false, Bytes: []uint8{0x0, 0x8c, 0xc3, 0x37, 0x92, 0x10, 0xec, 0x2c, 0x98}, FullBytes: []byte{2, 9, 0x0, 0x8c, 0xc3, 0x37, 0x92, 0x10, 0xec, 0x2c, 0x98}}, + SignatureAlgorithm: AlgorithmIdentifier{Algorithm: ObjectIdentifier{1, 2, 840, 113549, 1, 1, 5}}, + Issuer: RDNSequence{ + RelativeDistinguishedNameSET{AttributeTypeAndValue{Type: ObjectIdentifier{2, 5, 4, 6}, Value: "XX"}}, + RelativeDistinguishedNameSET{AttributeTypeAndValue{Type: ObjectIdentifier{2, 5, 4, 8}, Value: "Some-State"}}, + RelativeDistinguishedNameSET{AttributeTypeAndValue{Type: ObjectIdentifier{2, 5, 4, 7}, Value: "City"}}, + RelativeDistinguishedNameSET{AttributeTypeAndValue{Type: ObjectIdentifier{2, 5, 4, 10}, Value: "Internet Widgits Pty Ltd"}}, + RelativeDistinguishedNameSET{AttributeTypeAndValue{Type: ObjectIdentifier{2, 5, 4, 3}, Value: "false.example.com"}}, + RelativeDistinguishedNameSET{AttributeTypeAndValue{Type: ObjectIdentifier{1, 2, 840, 113549, 1, 9, 1}, Value: "false@example.com"}}, + }, + Validity: Validity{NotBefore: &time.Time{Year: 2009, Month: 10, Day: 8, Hour: 0, Minute: 25, Second: 53, Weekday: 0, ZoneOffset: 0, Zone: "UTC"}, NotAfter: &time.Time{Year: 2010, Month: 10, Day: 8, Hour: 0, Minute: 25, Second: 53, Weekday: 0, ZoneOffset: 0, Zone: "UTC"}}, + Subject: RDNSequence{ + RelativeDistinguishedNameSET{AttributeTypeAndValue{Type: ObjectIdentifier{2, 5, 4, 6}, Value: "XX"}}, + RelativeDistinguishedNameSET{AttributeTypeAndValue{Type: ObjectIdentifier{2, 5, 4, 8}, Value: "Some-State"}}, + RelativeDistinguishedNameSET{AttributeTypeAndValue{Type: ObjectIdentifier{2, 5, 4, 7}, Value: "City"}}, + RelativeDistinguishedNameSET{AttributeTypeAndValue{Type: ObjectIdentifier{2, 5, 4, 10}, Value: "Internet Widgits Pty Ltd"}}, + RelativeDistinguishedNameSET{AttributeTypeAndValue{Type: ObjectIdentifier{2, 5, 4, 3}, Value: "false.example.com"}}, + RelativeDistinguishedNameSET{AttributeTypeAndValue{Type: ObjectIdentifier{1, 2, 840, 113549, 1, 9, 1}, Value: "false@example.com"}}, + }, + PublicKey: PublicKeyInfo{ + Algorithm: AlgorithmIdentifier{Algorithm: ObjectIdentifier{1, 2, 840, 113549, 1, 1, 1}}, + PublicKey: BitString{ + Bytes: []uint8{ + 0x30, 0x48, 0x2, 0x41, 0x0, 0xcd, 0xb7, + 0x63, 0x9c, 0x32, 0x78, 0xf0, 0x6, 0xaa, 0x27, 0x7f, 0x6e, 0xaf, 0x42, + 0x90, 0x2b, 0x59, 0x2d, 0x8c, 0xbc, 0xbe, 0x38, 0xa1, 0xc9, 0x2b, 0xa4, + 0x69, 0x5a, 0x33, 0x1b, 0x1d, 0xea, 0xde, 0xad, 0xd8, 0xe9, 0xa5, 0xc2, + 0x7e, 0x8c, 0x4c, 0x2f, 0xd0, 0xa8, 0x88, 0x96, 0x57, 0x72, 0x2a, 0x4f, + 0x2a, 0xf7, 0x58, 0x9c, 0xf2, 0xc7, 0x70, 0x45, 0xdc, 0x8f, 0xde, 0xec, + 0x35, 0x7d, 0x2, 0x3, 0x1, 0x0, 0x1, + }, + BitLength: 592, + }, + }, + }, + SignatureAlgorithm: AlgorithmIdentifier{Algorithm: ObjectIdentifier{1, 2, 840, 113549, 1, 1, 5}}, + SignatureValue: BitString{ + Bytes: []uint8{ + 0xa6, 0x7b, 0x6, 0xec, 0x5e, 0xce, + 0x92, 0x77, 0x2c, 0xa4, 0x13, 0xcb, 0xa3, 0xca, 0x12, 0x56, 0x8f, 0xdc, 0x6c, + 0x7b, 0x45, 0x11, 0xcd, 0x40, 0xa7, 0xf6, 0x59, 0x98, 0x4, 0x2, 0xdf, 0x2b, + 0x99, 0x8b, 0xb9, 0xa4, 0xa8, 0xcb, 0xeb, 0x34, 0xc0, 0xf0, 0xa7, 0x8c, 0xf8, + 0xd9, 0x1e, 0xde, 0x14, 0xa5, 0xed, 0x76, 0xbf, 0x11, 0x6f, 0xe3, 0x60, 0xaa, + 0xfa, 0x88, 0x21, 0x49, 0x4, 0x35, + }, + BitLength: 512, + }, +} + +var derEncodedSelfSignedCertBytes = []byte{ + 0x30, 0x82, 0x02, 0x18, 0x30, + 0x82, 0x01, 0xc2, 0x02, 0x09, 0x00, 0x8c, 0xc3, 0x37, 0x92, 0x10, 0xec, 0x2c, + 0x98, 0x30, 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, + 0x05, 0x05, 0x00, 0x30, 0x81, 0x92, 0x31, 0x0b, 0x30, 0x09, 0x06, 0x03, 0x55, + 0x04, 0x06, 0x13, 0x02, 0x58, 0x58, 0x31, 0x13, 0x30, 0x11, 0x06, 0x03, 0x55, + 0x04, 0x08, 0x13, 0x0a, 0x53, 0x6f, 0x6d, 0x65, 0x2d, 0x53, 0x74, 0x61, 0x74, + 0x65, 0x31, 0x0d, 0x30, 0x0b, 0x06, 0x03, 0x55, 0x04, 0x07, 0x13, 0x04, 0x43, + 0x69, 0x74, 0x79, 0x31, 0x21, 0x30, 0x1f, 0x06, 0x03, 0x55, 0x04, 0x0a, 0x13, + 0x18, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x20, 0x57, 0x69, 0x64, + 0x67, 0x69, 0x74, 0x73, 0x20, 0x50, 0x74, 0x79, 0x20, 0x4c, 0x74, 0x64, 0x31, + 0x1a, 0x30, 0x18, 0x06, 0x03, 0x55, 0x04, 0x03, 0x13, 0x11, 0x66, 0x61, 0x6c, + 0x73, 0x65, 0x2e, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x2e, 0x63, 0x6f, + 0x6d, 0x31, 0x20, 0x30, 0x1e, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, + 0x01, 0x09, 0x01, 0x16, 0x11, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x40, 0x65, 0x78, + 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x30, 0x1e, 0x17, 0x0d, + 0x30, 0x39, 0x31, 0x30, 0x30, 0x38, 0x30, 0x30, 0x32, 0x35, 0x35, 0x33, 0x5a, + 0x17, 0x0d, 0x31, 0x30, 0x31, 0x30, 0x30, 0x38, 0x30, 0x30, 0x32, 0x35, 0x35, + 0x33, 0x5a, 0x30, 0x81, 0x92, 0x31, 0x0b, 0x30, 0x09, 0x06, 0x03, 0x55, 0x04, + 0x06, 0x13, 0x02, 0x58, 0x58, 0x31, 0x13, 0x30, 0x11, 0x06, 0x03, 0x55, 0x04, + 0x08, 0x13, 0x0a, 0x53, 0x6f, 0x6d, 0x65, 0x2d, 0x53, 0x74, 0x61, 0x74, 0x65, + 0x31, 0x0d, 0x30, 0x0b, 0x06, 0x03, 0x55, 0x04, 0x07, 0x13, 0x04, 0x43, 0x69, + 0x74, 0x79, 0x31, 0x21, 0x30, 0x1f, 0x06, 0x03, 0x55, 0x04, 0x0a, 0x13, 0x18, + 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x20, 0x57, 0x69, 0x64, 0x67, + 0x69, 0x74, 0x73, 0x20, 0x50, 0x74, 0x79, 0x20, 0x4c, 0x74, 0x64, 0x31, 0x1a, + 0x30, 0x18, 0x06, 0x03, 0x55, 0x04, 0x03, 0x13, 0x11, 0x66, 0x61, 0x6c, 0x73, + 0x65, 0x2e, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x2e, 0x63, 0x6f, 0x6d, + 0x31, 0x20, 0x30, 0x1e, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, + 0x09, 0x01, 0x16, 0x11, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x40, 0x65, 0x78, 0x61, + 0x6d, 0x70, 0x6c, 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x30, 0x5c, 0x30, 0x0d, 0x06, + 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x01, 0x05, 0x00, 0x03, + 0x4b, 0x00, 0x30, 0x48, 0x02, 0x41, 0x00, 0xcd, 0xb7, 0x63, 0x9c, 0x32, 0x78, + 0xf0, 0x06, 0xaa, 0x27, 0x7f, 0x6e, 0xaf, 0x42, 0x90, 0x2b, 0x59, 0x2d, 0x8c, + 0xbc, 0xbe, 0x38, 0xa1, 0xc9, 0x2b, 0xa4, 0x69, 0x5a, 0x33, 0x1b, 0x1d, 0xea, + 0xde, 0xad, 0xd8, 0xe9, 0xa5, 0xc2, 0x7e, 0x8c, 0x4c, 0x2f, 0xd0, 0xa8, 0x88, + 0x96, 0x57, 0x72, 0x2a, 0x4f, 0x2a, 0xf7, 0x58, 0x9c, 0xf2, 0xc7, 0x70, 0x45, + 0xdc, 0x8f, 0xde, 0xec, 0x35, 0x7d, 0x02, 0x03, 0x01, 0x00, 0x01, 0x30, 0x0d, + 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x05, 0x05, 0x00, + 0x03, 0x41, 0x00, 0xa6, 0x7b, 0x06, 0xec, 0x5e, 0xce, 0x92, 0x77, 0x2c, 0xa4, + 0x13, 0xcb, 0xa3, 0xca, 0x12, 0x56, 0x8f, 0xdc, 0x6c, 0x7b, 0x45, 0x11, 0xcd, + 0x40, 0xa7, 0xf6, 0x59, 0x98, 0x04, 0x02, 0xdf, 0x2b, 0x99, 0x8b, 0xb9, 0xa4, + 0xa8, 0xcb, 0xeb, 0x34, 0xc0, 0xf0, 0xa7, 0x8c, 0xf8, 0xd9, 0x1e, 0xde, 0x14, + 0xa5, 0xed, 0x76, 0xbf, 0x11, 0x6f, 0xe3, 0x60, 0xaa, 0xfa, 0x88, 0x21, 0x49, + 0x04, 0x35, +} + +var derEncodedPaypalNULCertBytes = []byte{ + 0x30, 0x82, 0x06, 0x44, 0x30, + 0x82, 0x05, 0xad, 0xa0, 0x03, 0x02, 0x01, 0x02, 0x02, 0x03, 0x00, 0xf0, 0x9b, + 0x30, 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x05, + 0x05, 0x00, 0x30, 0x82, 0x01, 0x12, 0x31, 0x0b, 0x30, 0x09, 0x06, 0x03, 0x55, + 0x04, 0x06, 0x13, 0x02, 0x45, 0x53, 0x31, 0x12, 0x30, 0x10, 0x06, 0x03, 0x55, + 0x04, 0x08, 0x13, 0x09, 0x42, 0x61, 0x72, 0x63, 0x65, 0x6c, 0x6f, 0x6e, 0x61, + 0x31, 0x12, 0x30, 0x10, 0x06, 0x03, 0x55, 0x04, 0x07, 0x13, 0x09, 0x42, 0x61, + 0x72, 0x63, 0x65, 0x6c, 0x6f, 0x6e, 0x61, 0x31, 0x29, 0x30, 0x27, 0x06, 0x03, + 0x55, 0x04, 0x0a, 0x13, 0x20, 0x49, 0x50, 0x53, 0x20, 0x43, 0x65, 0x72, 0x74, + 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x41, 0x75, 0x74, + 0x68, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x20, 0x73, 0x2e, 0x6c, 0x2e, 0x31, 0x2e, + 0x30, 0x2c, 0x06, 0x03, 0x55, 0x04, 0x0a, 0x14, 0x25, 0x67, 0x65, 0x6e, 0x65, + 0x72, 0x61, 0x6c, 0x40, 0x69, 0x70, 0x73, 0x63, 0x61, 0x2e, 0x63, 0x6f, 0x6d, + 0x20, 0x43, 0x2e, 0x49, 0x2e, 0x46, 0x2e, 0x20, 0x20, 0x42, 0x2d, 0x42, 0x36, + 0x32, 0x32, 0x31, 0x30, 0x36, 0x39, 0x35, 0x31, 0x2e, 0x30, 0x2c, 0x06, 0x03, + 0x55, 0x04, 0x0b, 0x13, 0x25, 0x69, 0x70, 0x73, 0x43, 0x41, 0x20, 0x43, 0x4c, + 0x41, 0x53, 0x45, 0x41, 0x31, 0x20, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, + 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, + 0x69, 0x74, 0x79, 0x31, 0x2e, 0x30, 0x2c, 0x06, 0x03, 0x55, 0x04, 0x03, 0x13, + 0x25, 0x69, 0x70, 0x73, 0x43, 0x41, 0x20, 0x43, 0x4c, 0x41, 0x53, 0x45, 0x41, + 0x31, 0x20, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x20, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x31, + 0x20, 0x30, 0x1e, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x09, + 0x01, 0x16, 0x11, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x6c, 0x40, 0x69, 0x70, + 0x73, 0x63, 0x61, 0x2e, 0x63, 0x6f, 0x6d, 0x30, 0x1e, 0x17, 0x0d, 0x30, 0x39, + 0x30, 0x32, 0x32, 0x34, 0x32, 0x33, 0x30, 0x34, 0x31, 0x37, 0x5a, 0x17, 0x0d, + 0x31, 0x31, 0x30, 0x32, 0x32, 0x34, 0x32, 0x33, 0x30, 0x34, 0x31, 0x37, 0x5a, + 0x30, 0x81, 0x94, 0x31, 0x0b, 0x30, 0x09, 0x06, 0x03, 0x55, 0x04, 0x06, 0x13, + 0x02, 0x55, 0x53, 0x31, 0x13, 0x30, 0x11, 0x06, 0x03, 0x55, 0x04, 0x08, 0x13, + 0x0a, 0x43, 0x61, 0x6c, 0x69, 0x66, 0x6f, 0x72, 0x6e, 0x69, 0x61, 0x31, 0x16, + 0x30, 0x14, 0x06, 0x03, 0x55, 0x04, 0x07, 0x13, 0x0d, 0x53, 0x61, 0x6e, 0x20, + 0x46, 0x72, 0x61, 0x6e, 0x63, 0x69, 0x73, 0x63, 0x6f, 0x31, 0x11, 0x30, 0x0f, + 0x06, 0x03, 0x55, 0x04, 0x0a, 0x13, 0x08, 0x53, 0x65, 0x63, 0x75, 0x72, 0x69, + 0x74, 0x79, 0x31, 0x14, 0x30, 0x12, 0x06, 0x03, 0x55, 0x04, 0x0b, 0x13, 0x0b, + 0x53, 0x65, 0x63, 0x75, 0x72, 0x65, 0x20, 0x55, 0x6e, 0x69, 0x74, 0x31, 0x2f, + 0x30, 0x2d, 0x06, 0x03, 0x55, 0x04, 0x03, 0x13, 0x26, 0x77, 0x77, 0x77, 0x2e, + 0x70, 0x61, 0x79, 0x70, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6d, 0x00, 0x73, 0x73, + 0x6c, 0x2e, 0x73, 0x65, 0x63, 0x75, 0x72, 0x65, 0x63, 0x6f, 0x6e, 0x6e, 0x65, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x63, 0x63, 0x30, 0x81, 0x9f, 0x30, 0x0d, + 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x01, 0x05, 0x00, + 0x03, 0x81, 0x8d, 0x00, 0x30, 0x81, 0x89, 0x02, 0x81, 0x81, 0x00, 0xd2, 0x69, + 0xfa, 0x6f, 0x3a, 0x00, 0xb4, 0x21, 0x1b, 0xc8, 0xb1, 0x02, 0xd7, 0x3f, 0x19, + 0xb2, 0xc4, 0x6d, 0xb4, 0x54, 0xf8, 0x8b, 0x8a, 0xcc, 0xdb, 0x72, 0xc2, 0x9e, + 0x3c, 0x60, 0xb9, 0xc6, 0x91, 0x3d, 0x82, 0xb7, 0x7d, 0x99, 0xff, 0xd1, 0x29, + 0x84, 0xc1, 0x73, 0x53, 0x9c, 0x82, 0xdd, 0xfc, 0x24, 0x8c, 0x77, 0xd5, 0x41, + 0xf3, 0xe8, 0x1e, 0x42, 0xa1, 0xad, 0x2d, 0x9e, 0xff, 0x5b, 0x10, 0x26, 0xce, + 0x9d, 0x57, 0x17, 0x73, 0x16, 0x23, 0x38, 0xc8, 0xd6, 0xf1, 0xba, 0xa3, 0x96, + 0x5b, 0x16, 0x67, 0x4a, 0x4f, 0x73, 0x97, 0x3a, 0x4d, 0x14, 0xa4, 0xf4, 0xe2, + 0x3f, 0x8b, 0x05, 0x83, 0x42, 0xd1, 0xd0, 0xdc, 0x2f, 0x7a, 0xe5, 0xb6, 0x10, + 0xb2, 0x11, 0xc0, 0xdc, 0x21, 0x2a, 0x90, 0xff, 0xae, 0x97, 0x71, 0x5a, 0x49, + 0x81, 0xac, 0x40, 0xf3, 0x3b, 0xb8, 0x59, 0xb2, 0x4f, 0x02, 0x03, 0x01, 0x00, + 0x01, 0xa3, 0x82, 0x03, 0x21, 0x30, 0x82, 0x03, 0x1d, 0x30, 0x09, 0x06, 0x03, + 0x55, 0x1d, 0x13, 0x04, 0x02, 0x30, 0x00, 0x30, 0x11, 0x06, 0x09, 0x60, 0x86, + 0x48, 0x01, 0x86, 0xf8, 0x42, 0x01, 0x01, 0x04, 0x04, 0x03, 0x02, 0x06, 0x40, + 0x30, 0x0b, 0x06, 0x03, 0x55, 0x1d, 0x0f, 0x04, 0x04, 0x03, 0x02, 0x03, 0xf8, + 0x30, 0x13, 0x06, 0x03, 0x55, 0x1d, 0x25, 0x04, 0x0c, 0x30, 0x0a, 0x06, 0x08, + 0x2b, 0x06, 0x01, 0x05, 0x05, 0x07, 0x03, 0x01, 0x30, 0x1d, 0x06, 0x03, 0x55, + 0x1d, 0x0e, 0x04, 0x16, 0x04, 0x14, 0x61, 0x8f, 0x61, 0x34, 0x43, 0x55, 0x14, + 0x7f, 0x27, 0x09, 0xce, 0x4c, 0x8b, 0xea, 0x9b, 0x7b, 0x19, 0x25, 0xbc, 0x6e, + 0x30, 0x1f, 0x06, 0x03, 0x55, 0x1d, 0x23, 0x04, 0x18, 0x30, 0x16, 0x80, 0x14, + 0x0e, 0x07, 0x60, 0xd4, 0x39, 0xc9, 0x1b, 0x5b, 0x5d, 0x90, 0x7b, 0x23, 0xc8, + 0xd2, 0x34, 0x9d, 0x4a, 0x9a, 0x46, 0x39, 0x30, 0x09, 0x06, 0x03, 0x55, 0x1d, + 0x11, 0x04, 0x02, 0x30, 0x00, 0x30, 0x1c, 0x06, 0x03, 0x55, 0x1d, 0x12, 0x04, + 0x15, 0x30, 0x13, 0x81, 0x11, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x6c, 0x40, + 0x69, 0x70, 0x73, 0x63, 0x61, 0x2e, 0x63, 0x6f, 0x6d, 0x30, 0x72, 0x06, 0x09, + 0x60, 0x86, 0x48, 0x01, 0x86, 0xf8, 0x42, 0x01, 0x0d, 0x04, 0x65, 0x16, 0x63, + 0x4f, 0x72, 0x67, 0x61, 0x6e, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x20, + 0x49, 0x6e, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x4e, + 0x4f, 0x54, 0x20, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x41, 0x54, 0x45, 0x44, 0x2e, + 0x20, 0x43, 0x4c, 0x41, 0x53, 0x45, 0x41, 0x31, 0x20, 0x53, 0x65, 0x72, 0x76, + 0x65, 0x72, 0x20, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, + 0x65, 0x20, 0x69, 0x73, 0x73, 0x75, 0x65, 0x64, 0x20, 0x62, 0x79, 0x20, 0x68, + 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x77, 0x77, 0x77, 0x2e, 0x69, 0x70, + 0x73, 0x63, 0x61, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x30, 0x2f, 0x06, 0x09, 0x60, + 0x86, 0x48, 0x01, 0x86, 0xf8, 0x42, 0x01, 0x02, 0x04, 0x22, 0x16, 0x20, 0x68, + 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x77, 0x77, 0x77, 0x2e, 0x69, 0x70, + 0x73, 0x63, 0x61, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x69, 0x70, 0x73, 0x63, 0x61, + 0x32, 0x30, 0x30, 0x32, 0x2f, 0x30, 0x43, 0x06, 0x09, 0x60, 0x86, 0x48, 0x01, + 0x86, 0xf8, 0x42, 0x01, 0x04, 0x04, 0x36, 0x16, 0x34, 0x68, 0x74, 0x74, 0x70, + 0x73, 0x3a, 0x2f, 0x2f, 0x77, 0x77, 0x77, 0x2e, 0x69, 0x70, 0x73, 0x63, 0x61, + 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x69, 0x70, 0x73, 0x63, 0x61, 0x32, 0x30, 0x30, + 0x32, 0x2f, 0x69, 0x70, 0x73, 0x63, 0x61, 0x32, 0x30, 0x30, 0x32, 0x43, 0x4c, + 0x41, 0x53, 0x45, 0x41, 0x31, 0x2e, 0x63, 0x72, 0x6c, 0x30, 0x46, 0x06, 0x09, + 0x60, 0x86, 0x48, 0x01, 0x86, 0xf8, 0x42, 0x01, 0x03, 0x04, 0x39, 0x16, 0x37, + 0x68, 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x77, 0x77, 0x77, 0x2e, 0x69, + 0x70, 0x73, 0x63, 0x61, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x69, 0x70, 0x73, 0x63, + 0x61, 0x32, 0x30, 0x30, 0x32, 0x2f, 0x72, 0x65, 0x76, 0x6f, 0x63, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x43, 0x4c, 0x41, 0x53, 0x45, 0x41, 0x31, 0x2e, 0x68, 0x74, + 0x6d, 0x6c, 0x3f, 0x30, 0x43, 0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x86, 0xf8, + 0x42, 0x01, 0x07, 0x04, 0x36, 0x16, 0x34, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3a, + 0x2f, 0x2f, 0x77, 0x77, 0x77, 0x2e, 0x69, 0x70, 0x73, 0x63, 0x61, 0x2e, 0x63, + 0x6f, 0x6d, 0x2f, 0x69, 0x70, 0x73, 0x63, 0x61, 0x32, 0x30, 0x30, 0x32, 0x2f, + 0x72, 0x65, 0x6e, 0x65, 0x77, 0x61, 0x6c, 0x43, 0x4c, 0x41, 0x53, 0x45, 0x41, + 0x31, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x3f, 0x30, 0x41, 0x06, 0x09, 0x60, 0x86, + 0x48, 0x01, 0x86, 0xf8, 0x42, 0x01, 0x08, 0x04, 0x34, 0x16, 0x32, 0x68, 0x74, + 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x77, 0x77, 0x77, 0x2e, 0x69, 0x70, 0x73, + 0x63, 0x61, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x69, 0x70, 0x73, 0x63, 0x61, 0x32, + 0x30, 0x30, 0x32, 0x2f, 0x70, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x43, 0x4c, 0x41, + 0x53, 0x45, 0x41, 0x31, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x30, 0x81, 0x83, 0x06, + 0x03, 0x55, 0x1d, 0x1f, 0x04, 0x7c, 0x30, 0x7a, 0x30, 0x39, 0xa0, 0x37, 0xa0, + 0x35, 0x86, 0x33, 0x68, 0x74, 0x74, 0x70, 0x3a, 0x2f, 0x2f, 0x77, 0x77, 0x77, + 0x2e, 0x69, 0x70, 0x73, 0x63, 0x61, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x69, 0x70, + 0x73, 0x63, 0x61, 0x32, 0x30, 0x30, 0x32, 0x2f, 0x69, 0x70, 0x73, 0x63, 0x61, + 0x32, 0x30, 0x30, 0x32, 0x43, 0x4c, 0x41, 0x53, 0x45, 0x41, 0x31, 0x2e, 0x63, + 0x72, 0x6c, 0x30, 0x3d, 0xa0, 0x3b, 0xa0, 0x39, 0x86, 0x37, 0x68, 0x74, 0x74, + 0x70, 0x3a, 0x2f, 0x2f, 0x77, 0x77, 0x77, 0x62, 0x61, 0x63, 0x6b, 0x2e, 0x69, + 0x70, 0x73, 0x63, 0x61, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x69, 0x70, 0x73, 0x63, + 0x61, 0x32, 0x30, 0x30, 0x32, 0x2f, 0x69, 0x70, 0x73, 0x63, 0x61, 0x32, 0x30, + 0x30, 0x32, 0x43, 0x4c, 0x41, 0x53, 0x45, 0x41, 0x31, 0x2e, 0x63, 0x72, 0x6c, + 0x30, 0x32, 0x06, 0x08, 0x2b, 0x06, 0x01, 0x05, 0x05, 0x07, 0x01, 0x01, 0x04, + 0x26, 0x30, 0x24, 0x30, 0x22, 0x06, 0x08, 0x2b, 0x06, 0x01, 0x05, 0x05, 0x07, + 0x30, 0x01, 0x86, 0x16, 0x68, 0x74, 0x74, 0x70, 0x3a, 0x2f, 0x2f, 0x6f, 0x63, + 0x73, 0x70, 0x2e, 0x69, 0x70, 0x73, 0x63, 0x61, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, + 0x30, 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x05, + 0x05, 0x00, 0x03, 0x81, 0x81, 0x00, 0x68, 0xee, 0x79, 0x97, 0x97, 0xdd, 0x3b, + 0xef, 0x16, 0x6a, 0x06, 0xf2, 0x14, 0x9a, 0x6e, 0xcd, 0x9e, 0x12, 0xf7, 0xaa, + 0x83, 0x10, 0xbd, 0xd1, 0x7c, 0x98, 0xfa, 0xc7, 0xae, 0xd4, 0x0e, 0x2c, 0x9e, + 0x38, 0x05, 0x9d, 0x52, 0x60, 0xa9, 0x99, 0x0a, 0x81, 0xb4, 0x98, 0x90, 0x1d, + 0xae, 0xbb, 0x4a, 0xd7, 0xb9, 0xdc, 0x88, 0x9e, 0x37, 0x78, 0x41, 0x5b, 0xf7, + 0x82, 0xa5, 0xf2, 0xba, 0x41, 0x25, 0x5a, 0x90, 0x1a, 0x1e, 0x45, 0x38, 0xa1, + 0x52, 0x58, 0x75, 0x94, 0x26, 0x44, 0xfb, 0x20, 0x07, 0xba, 0x44, 0xcc, 0xe5, + 0x4a, 0x2d, 0x72, 0x3f, 0x98, 0x47, 0xf6, 0x26, 0xdc, 0x05, 0x46, 0x05, 0x07, + 0x63, 0x21, 0xab, 0x46, 0x9b, 0x9c, 0x78, 0xd5, 0x54, 0x5b, 0x3d, 0x0c, 0x1e, + 0xc8, 0x64, 0x8c, 0xb5, 0x50, 0x23, 0x82, 0x6f, 0xdb, 0xb8, 0x22, 0x1c, 0x43, + 0x96, 0x07, 0xa8, 0xbb, +} diff --git a/libgo/go/asn1/common.go b/libgo/go/asn1/common.go new file mode 100644 index 000000000..4a5eca145 --- /dev/null +++ b/libgo/go/asn1/common.go @@ -0,0 +1,149 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package asn1 + +import ( + "reflect" + "strconv" + "strings" +) + +// ASN.1 objects have metadata preceeding them: +// the tag: the type of the object +// a flag denoting if this object is compound or not +// the class type: the namespace of the tag +// the length of the object, in bytes + +// Here are some standard tags and classes + +const ( + tagBoolean = 1 + tagInteger = 2 + tagBitString = 3 + tagOctetString = 4 + tagOID = 6 + tagEnum = 10 + tagSequence = 16 + tagSet = 17 + tagPrintableString = 19 + tagT61String = 20 + tagIA5String = 22 + tagUTCTime = 23 + tagGeneralizedTime = 24 +) + +const ( + classUniversal = 0 + classApplication = 1 + classContextSpecific = 2 + classPrivate = 3 +) + +type tagAndLength struct { + class, tag, length int + isCompound bool +} + +// ASN.1 has IMPLICIT and EXPLICIT tags, which can be translated as "instead +// of" and "in addition to". When not specified, every primitive type has a +// default tag in the UNIVERSAL class. +// +// For example: a BIT STRING is tagged [UNIVERSAL 3] by default (although ASN.1 +// doesn't actually have a UNIVERSAL keyword). However, by saying [IMPLICIT +// CONTEXT-SPECIFIC 42], that means that the tag is replaced by another. +// +// On the other hand, if it said [EXPLICIT CONTEXT-SPECIFIC 10], then an +// /additional/ tag would wrap the default tag. This explicit tag will have the +// compound flag set. +// +// (This is used in order to remove ambiguity with optional elements.) +// +// You can layer EXPLICIT and IMPLICIT tags to an arbitrary depth, however we +// don't support that here. We support a single layer of EXPLICIT or IMPLICIT +// tagging with tag strings on the fields of a structure. + +// fieldParameters is the parsed representation of tag string from a structure field. +type fieldParameters struct { + optional bool // true iff the field is OPTIONAL + explicit bool // true iff and EXPLICIT tag is in use. + defaultValue *int64 // a default value for INTEGER typed fields (maybe nil). + tag *int // the EXPLICIT or IMPLICIT tag (maybe nil). + stringType int // the string tag to use when marshaling. + set bool // true iff this should be encoded as a SET + + // Invariants: + // if explicit is set, tag is non-nil. +} + +// Given a tag string with the format specified in the package comment, +// parseFieldParameters will parse it into a fieldParameters structure, +// ignoring unknown parts of the string. +func parseFieldParameters(str string) (ret fieldParameters) { + for _, part := range strings.Split(str, ",", -1) { + switch { + case part == "optional": + ret.optional = true + case part == "explicit": + ret.explicit = true + if ret.tag == nil { + ret.tag = new(int) + *ret.tag = 0 + } + case part == "ia5": + ret.stringType = tagIA5String + case part == "printable": + ret.stringType = tagPrintableString + case strings.HasPrefix(part, "default:"): + i, err := strconv.Atoi64(part[8:]) + if err == nil { + ret.defaultValue = new(int64) + *ret.defaultValue = i + } + case strings.HasPrefix(part, "tag:"): + i, err := strconv.Atoi(part[4:]) + if err == nil { + ret.tag = new(int) + *ret.tag = i + } + case part == "set": + ret.set = true + } + } + return +} + +// Given a reflected Go type, getUniversalType returns the default tag number +// and expected compound flag. +func getUniversalType(t reflect.Type) (tagNumber int, isCompound, ok bool) { + switch t { + case objectIdentifierType: + return tagOID, false, true + case bitStringType: + return tagBitString, false, true + case timeType: + return tagUTCTime, false, true + case enumeratedType: + return tagEnum, false, true + } + switch t := t.(type) { + case *reflect.BoolType: + return tagBoolean, false, true + case *reflect.IntType: + return tagInteger, false, true + case *reflect.StructType: + return tagSequence, true, true + case *reflect.SliceType: + if t.Elem().Kind() == reflect.Uint8 { + return tagOctetString, false, true + } + if strings.HasSuffix(t.Name(), "SET") { + return tagSet, true, true + } + return tagSequence, true, true + case *reflect.StringType: + return tagPrintableString, false, true + } + return 0, false, false +} diff --git a/libgo/go/asn1/marshal.go b/libgo/go/asn1/marshal.go new file mode 100644 index 000000000..24548714b --- /dev/null +++ b/libgo/go/asn1/marshal.go @@ -0,0 +1,482 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package asn1 + +import ( + "bytes" + "fmt" + "io" + "os" + "reflect" + "time" +) + +// A forkableWriter is an in-memory buffer that can be +// 'forked' to create new forkableWriters that bracket the +// original. After +// pre, post := w.fork(); +// the overall sequence of bytes represented is logically w+pre+post. +type forkableWriter struct { + *bytes.Buffer + pre, post *forkableWriter +} + +func newForkableWriter() *forkableWriter { + return &forkableWriter{bytes.NewBuffer(nil), nil, nil} +} + +func (f *forkableWriter) fork() (pre, post *forkableWriter) { + if f.pre != nil || f.post != nil { + panic("have already forked") + } + f.pre = newForkableWriter() + f.post = newForkableWriter() + return f.pre, f.post +} + +func (f *forkableWriter) Len() (l int) { + l += f.Buffer.Len() + if f.pre != nil { + l += f.pre.Len() + } + if f.post != nil { + l += f.post.Len() + } + return +} + +func (f *forkableWriter) writeTo(out io.Writer) (n int, err os.Error) { + n, err = out.Write(f.Bytes()) + if err != nil { + return + } + + var nn int + + if f.pre != nil { + nn, err = f.pre.writeTo(out) + n += nn + if err != nil { + return + } + } + + if f.post != nil { + nn, err = f.post.writeTo(out) + n += nn + } + return +} + +func marshalBase128Int(out *forkableWriter, n int64) (err os.Error) { + if n == 0 { + err = out.WriteByte(0) + return + } + + l := 0 + for i := n; i > 0; i >>= 7 { + l++ + } + + for i := l - 1; i >= 0; i-- { + o := byte(n >> uint(i*7)) + o &= 0x7f + if i != 0 { + o |= 0x80 + } + err = out.WriteByte(o) + if err != nil { + return + } + } + + return nil +} + +func marshalInt64(out *forkableWriter, i int64) (err os.Error) { + n := int64Length(i) + + for ; n > 0; n-- { + err = out.WriteByte(byte(i >> uint((n-1)*8))) + if err != nil { + return + } + } + + return nil +} + +func int64Length(i int64) (numBytes int) { + numBytes = 1 + + for i > 127 { + numBytes++ + i >>= 8 + } + + for i < -128 { + numBytes++ + i >>= 8 + } + + return +} + +func marshalTagAndLength(out *forkableWriter, t tagAndLength) (err os.Error) { + b := uint8(t.class) << 6 + if t.isCompound { + b |= 0x20 + } + if t.tag >= 31 { + b |= 0x1f + err = out.WriteByte(b) + if err != nil { + return + } + err = marshalBase128Int(out, int64(t.tag)) + if err != nil { + return + } + } else { + b |= uint8(t.tag) + err = out.WriteByte(b) + if err != nil { + return + } + } + + if t.length >= 128 { + l := int64Length(int64(t.length)) + err = out.WriteByte(0x80 | byte(l)) + if err != nil { + return + } + err = marshalInt64(out, int64(t.length)) + if err != nil { + return + } + } else { + err = out.WriteByte(byte(t.length)) + if err != nil { + return + } + } + + return nil +} + +func marshalBitString(out *forkableWriter, b BitString) (err os.Error) { + paddingBits := byte((8 - b.BitLength%8) % 8) + err = out.WriteByte(paddingBits) + if err != nil { + return + } + _, err = out.Write(b.Bytes) + return +} + +func marshalObjectIdentifier(out *forkableWriter, oid []int) (err os.Error) { + if len(oid) < 2 || oid[0] > 6 || oid[1] >= 40 { + return StructuralError{"invalid object identifier"} + } + + err = out.WriteByte(byte(oid[0]*40 + oid[1])) + if err != nil { + return + } + for i := 2; i < len(oid); i++ { + err = marshalBase128Int(out, int64(oid[i])) + if err != nil { + return + } + } + + return +} + +func marshalPrintableString(out *forkableWriter, s string) (err os.Error) { + b := []byte(s) + for _, c := range b { + if !isPrintable(c) { + return StructuralError{"PrintableString contains invalid character"} + } + } + + _, err = out.Write(b) + return +} + +func marshalIA5String(out *forkableWriter, s string) (err os.Error) { + b := []byte(s) + for _, c := range b { + if c > 127 { + return StructuralError{"IA5String contains invalid character"} + } + } + + _, err = out.Write(b) + return +} + +func marshalTwoDigits(out *forkableWriter, v int) (err os.Error) { + err = out.WriteByte(byte('0' + (v/10)%10)) + if err != nil { + return + } + return out.WriteByte(byte('0' + v%10)) +} + +func marshalUTCTime(out *forkableWriter, t *time.Time) (err os.Error) { + switch { + case 1950 <= t.Year && t.Year < 2000: + err = marshalTwoDigits(out, int(t.Year-1900)) + case 2000 <= t.Year && t.Year < 2050: + err = marshalTwoDigits(out, int(t.Year-2000)) + default: + return StructuralError{"Cannot represent time as UTCTime"} + } + + if err != nil { + return + } + + err = marshalTwoDigits(out, t.Month) + if err != nil { + return + } + + err = marshalTwoDigits(out, t.Day) + if err != nil { + return + } + + err = marshalTwoDigits(out, t.Hour) + if err != nil { + return + } + + err = marshalTwoDigits(out, t.Minute) + if err != nil { + return + } + + err = marshalTwoDigits(out, t.Second) + if err != nil { + return + } + + switch { + case t.ZoneOffset/60 == 0: + err = out.WriteByte('Z') + return + case t.ZoneOffset > 0: + err = out.WriteByte('+') + case t.ZoneOffset < 0: + err = out.WriteByte('-') + } + + if err != nil { + return + } + + offsetMinutes := t.ZoneOffset / 60 + if offsetMinutes < 0 { + offsetMinutes = -offsetMinutes + } + + err = marshalTwoDigits(out, offsetMinutes/60) + if err != nil { + return + } + + err = marshalTwoDigits(out, offsetMinutes%60) + return +} + +func stripTagAndLength(in []byte) []byte { + _, offset, err := parseTagAndLength(in, 0) + if err != nil { + return in + } + return in[offset:] +} + +func marshalBody(out *forkableWriter, value reflect.Value, params fieldParameters) (err os.Error) { + switch value.Type() { + case timeType: + return marshalUTCTime(out, value.Interface().(*time.Time)) + case bitStringType: + return marshalBitString(out, value.Interface().(BitString)) + case objectIdentifierType: + return marshalObjectIdentifier(out, value.Interface().(ObjectIdentifier)) + } + + switch v := value.(type) { + case *reflect.BoolValue: + if v.Get() { + return out.WriteByte(1) + } else { + return out.WriteByte(0) + } + case *reflect.IntValue: + return marshalInt64(out, int64(v.Get())) + case *reflect.StructValue: + t := v.Type().(*reflect.StructType) + + startingField := 0 + + // If the first element of the structure is a non-empty + // RawContents, then we don't bother serialising the rest. + if t.NumField() > 0 && t.Field(0).Type == rawContentsType { + s := v.Field(0).(*reflect.SliceValue) + if s.Len() > 0 { + bytes := make([]byte, s.Len()) + for i := 0; i < s.Len(); i++ { + bytes[i] = uint8(s.Elem(i).(*reflect.UintValue).Get()) + } + /* The RawContents will contain the tag and + * length fields but we'll also be writing + * those outselves, so we strip them out of + * bytes */ + _, err = out.Write(stripTagAndLength(bytes)) + return + } else { + startingField = 1 + } + } + + for i := startingField; i < t.NumField(); i++ { + var pre *forkableWriter + pre, out = out.fork() + err = marshalField(pre, v.Field(i), parseFieldParameters(t.Field(i).Tag)) + if err != nil { + return + } + } + return + case *reflect.SliceValue: + sliceType := v.Type().(*reflect.SliceType) + if sliceType.Elem().Kind() == reflect.Uint8 { + bytes := make([]byte, v.Len()) + for i := 0; i < v.Len(); i++ { + bytes[i] = uint8(v.Elem(i).(*reflect.UintValue).Get()) + } + _, err = out.Write(bytes) + return + } + + var params fieldParameters + for i := 0; i < v.Len(); i++ { + var pre *forkableWriter + pre, out = out.fork() + err = marshalField(pre, v.Elem(i), params) + if err != nil { + return + } + } + return + case *reflect.StringValue: + if params.stringType == tagIA5String { + return marshalIA5String(out, v.Get()) + } else { + return marshalPrintableString(out, v.Get()) + } + return + } + + return StructuralError{"unknown Go type"} +} + +func marshalField(out *forkableWriter, v reflect.Value, params fieldParameters) (err os.Error) { + // If the field is an interface{} then recurse into it. + if v, ok := v.(*reflect.InterfaceValue); ok && v.Type().(*reflect.InterfaceType).NumMethod() == 0 { + return marshalField(out, v.Elem(), params) + } + + if v.Type() == rawValueType { + rv := v.Interface().(RawValue) + err = marshalTagAndLength(out, tagAndLength{rv.Class, rv.Tag, len(rv.Bytes), rv.IsCompound}) + if err != nil { + return + } + _, err = out.Write(rv.Bytes) + return + } + + if params.optional && reflect.DeepEqual(v.Interface(), reflect.MakeZero(v.Type()).Interface()) { + return + } + + tag, isCompound, ok := getUniversalType(v.Type()) + if !ok { + err = StructuralError{fmt.Sprintf("unknown Go type: %v", v.Type())} + return + } + class := classUniversal + + if params.stringType != 0 { + if tag != tagPrintableString { + return StructuralError{"Explicit string type given to non-string member"} + } + tag = params.stringType + } + + if params.set { + if tag != tagSequence { + return StructuralError{"Non sequence tagged as set"} + } + tag = tagSet + } + + tags, body := out.fork() + + err = marshalBody(body, v, params) + if err != nil { + return + } + + bodyLen := body.Len() + + var explicitTag *forkableWriter + if params.explicit { + explicitTag, tags = tags.fork() + } + + if !params.explicit && params.tag != nil { + // implicit tag. + tag = *params.tag + class = classContextSpecific + } + + err = marshalTagAndLength(tags, tagAndLength{class, tag, bodyLen, isCompound}) + if err != nil { + return + } + + if params.explicit { + err = marshalTagAndLength(explicitTag, tagAndLength{ + class: classContextSpecific, + tag: *params.tag, + length: bodyLen + tags.Len(), + isCompound: true, + }) + } + + return nil +} + +// Marshal returns the ASN.1 encoding of val. +func Marshal(val interface{}) ([]byte, os.Error) { + var out bytes.Buffer + v := reflect.NewValue(val) + f := newForkableWriter() + err := marshalField(f, v, fieldParameters{}) + if err != nil { + return nil, err + } + _, err = f.writeTo(&out) + return out.Bytes(), nil +} diff --git a/libgo/go/asn1/marshal_test.go b/libgo/go/asn1/marshal_test.go new file mode 100644 index 000000000..85eafc9e4 --- /dev/null +++ b/libgo/go/asn1/marshal_test.go @@ -0,0 +1,100 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package asn1 + +import ( + "bytes" + "encoding/hex" + "testing" + "time" +) + +type intStruct struct { + A int +} + +type twoIntStruct struct { + A int + B int +} + +type nestedStruct struct { + A intStruct +} + +type rawContentsStruct struct { + Raw RawContent + A int +} + +type implicitTagTest struct { + A int "implicit,tag:5" +} + +type explicitTagTest struct { + A int "explicit,tag:5" +} + +type ia5StringTest struct { + A string "ia5" +} + +type printableStringTest struct { + A string "printable" +} + +type testSET []int + +func setPST(t *time.Time) *time.Time { + t.ZoneOffset = -28800 + return t +} + +type marshalTest struct { + in interface{} + out string // hex encoded +} + +var marshalTests = []marshalTest{ + {10, "02010a"}, + {127, "02017f"}, + {128, "02020080"}, + {-128, "020180"}, + {-129, "0202ff7f"}, + {intStruct{64}, "3003020140"}, + {twoIntStruct{64, 65}, "3006020140020141"}, + {nestedStruct{intStruct{127}}, "3005300302017f"}, + {[]byte{1, 2, 3}, "0403010203"}, + {implicitTagTest{64}, "3003850140"}, + {explicitTagTest{64}, "3005a503020140"}, + {time.SecondsToUTC(0), "170d3730303130313030303030305a"}, + {time.SecondsToUTC(1258325776), "170d3039313131353232353631365a"}, + {setPST(time.SecondsToUTC(1258325776)), "17113039313131353232353631362d30383030"}, + {BitString{[]byte{0x80}, 1}, "03020780"}, + {BitString{[]byte{0x81, 0xf0}, 12}, "03030481f0"}, + {ObjectIdentifier([]int{1, 2, 3, 4}), "06032a0304"}, + {ObjectIdentifier([]int{1, 2, 840, 133549, 1, 1, 5}), "06092a864888932d010105"}, + {"test", "130474657374"}, + {ia5StringTest{"test"}, "3006160474657374"}, + {printableStringTest{"test"}, "3006130474657374"}, + {printableStringTest{"test*"}, "30071305746573742a"}, + {rawContentsStruct{nil, 64}, "3003020140"}, + {rawContentsStruct{[]byte{0x30, 3, 1, 2, 3}, 64}, "3003010203"}, + {RawValue{Tag: 1, Class: 2, IsCompound: false, Bytes: []byte{1, 2, 3}}, "8103010203"}, + {testSET([]int{10}), "310302010a"}, +} + +func TestMarshal(t *testing.T) { + for i, test := range marshalTests { + data, err := Marshal(test.in) + if err != nil { + t.Errorf("#%d failed: %s", i, err) + } + out, _ := hex.DecodeString(test.out) + if bytes.Compare(out, data) != 0 { + t.Errorf("#%d got: %x want %x", i, data, out) + } + } +} diff --git a/libgo/go/big/arith.go b/libgo/go/big/arith.go new file mode 100644 index 000000000..a4048d6d7 --- /dev/null +++ b/libgo/go/big/arith.go @@ -0,0 +1,259 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file provides Go implementations of elementary multi-precision +// arithmetic operations on word vectors. Needed for platforms without +// assembly implementations of these routines. + +package big + +// TODO(gri) Decide if Word needs to remain exported. + +type Word uintptr + +const ( + // Compute the size _S of a Word in bytes. + _m = ^Word(0) + _logS = _m>>8&1 + _m>>16&1 + _m>>32&1 + _S = 1 << _logS + + _W = _S << 3 // word size in bits + _B = 1 << _W // digit base + _M = _B - 1 // digit mask + + _W2 = _W / 2 // half word size in bits + _B2 = 1 << _W2 // half digit base + _M2 = _B2 - 1 // half digit mask +) + + +// ---------------------------------------------------------------------------- +// Elementary operations on words +// +// These operations are used by the vector operations below. + +// z1<<_W + z0 = x+y+c, with c == 0 or 1 +func addWW_g(x, y, c Word) (z1, z0 Word) { + yc := y + c + z0 = x + yc + if z0 < x || yc < y { + z1 = 1 + } + return +} + + +// z1<<_W + z0 = x-y-c, with c == 0 or 1 +func subWW_g(x, y, c Word) (z1, z0 Word) { + yc := y + c + z0 = x - yc + if z0 > x || yc < y { + z1 = 1 + } + return +} + + +// z1<<_W + z0 = x*y +func mulWW(x, y Word) (z1, z0 Word) { return mulWW_g(x, y) } +// Adapted from Warren, Hacker's Delight, p. 132. +func mulWW_g(x, y Word) (z1, z0 Word) { + x0 := x & _M2 + x1 := x >> _W2 + y0 := y & _M2 + y1 := y >> _W2 + w0 := x0 * y0 + t := x1*y0 + w0>>_W2 + w1 := t & _M2 + w2 := t >> _W2 + w1 += x0 * y1 + z1 = x1*y1 + w2 + w1>>_W2 + z0 = x * y + return +} + + +// z1<<_W + z0 = x*y + c +func mulAddWWW_g(x, y, c Word) (z1, z0 Word) { + z1, zz0 := mulWW(x, y) + if z0 = zz0 + c; z0 < zz0 { + z1++ + } + return +} + + +// Length of x in bits. +func bitLen(x Word) (n int) { + for ; x >= 0x100; x >>= 8 { + n += 8 + } + for ; x > 0; x >>= 1 { + n++ + } + return +} + + +// log2 computes the integer binary logarithm of x. +// The result is the integer n for which 2^n <= x < 2^(n+1). +// If x == 0, the result is -1. +func log2(x Word) int { + return bitLen(x) - 1 +} + + +// Number of leading zeros in x. +func leadingZeros(x Word) uint { + return uint(_W - bitLen(x)) +} + + +// q = (u1<<_W + u0 - r)/y +func divWW(x1, x0, y Word) (q, r Word) { return divWW_g(x1, x0, y) } +// Adapted from Warren, Hacker's Delight, p. 152. +func divWW_g(u1, u0, v Word) (q, r Word) { + if u1 >= v { + return 1<<_W - 1, 1<<_W - 1 + } + + s := leadingZeros(v) + v <<= s + + vn1 := v >> _W2 + vn0 := v & _M2 + un32 := u1<>(_W-s) + un10 := u0 << s + un1 := un10 >> _W2 + un0 := un10 & _M2 + q1 := un32 / vn1 + rhat := un32 - q1*vn1 + +again1: + if q1 >= _B2 || q1*vn0 > _B2*rhat+un1 { + q1-- + rhat += vn1 + if rhat < _B2 { + goto again1 + } + } + + un21 := un32*_B2 + un1 - q1*v + q0 := un21 / vn1 + rhat = un21 - q0*vn1 + +again2: + if q0 >= _B2 || q0*vn0 > _B2*rhat+un0 { + q0-- + rhat += vn1 + if rhat < _B2 { + goto again2 + } + } + + return q1*_B2 + q0, (un21*_B2 + un0 - q0*v) >> s +} + + +func addVV(z, x, y []Word) (c Word) { return addVV_g(z, x, y) } +func addVV_g(z, x, y []Word) (c Word) { + for i := range z { + c, z[i] = addWW_g(x[i], y[i], c) + } + return +} + + +func subVV(z, x, y []Word) (c Word) { return subVV_g(z, x, y) } +func subVV_g(z, x, y []Word) (c Word) { + for i := range z { + c, z[i] = subWW_g(x[i], y[i], c) + } + return +} + + +func addVW(z, x []Word, y Word) (c Word) { return addVW_g(z, x, y) } +func addVW_g(z, x []Word, y Word) (c Word) { + c = y + for i := range z { + c, z[i] = addWW_g(x[i], c, 0) + } + return +} + + +func subVW(z, x []Word, y Word) (c Word) { return subVW_g(z, x, y) } +func subVW_g(z, x []Word, y Word) (c Word) { + c = y + for i := range z { + c, z[i] = subWW_g(x[i], c, 0) + } + return +} + + +func shlVW(z, x []Word, s Word) (c Word) { return shlVW_g(z, x, s) } +func shlVW_g(z, x []Word, s Word) (c Word) { + if n := len(z); n > 0 { + ŝ := _W - s + w1 := x[n-1] + c = w1 >> ŝ + for i := n - 1; i > 0; i-- { + w := w1 + w1 = x[i-1] + z[i] = w<>ŝ + } + z[0] = w1 << s + } + return +} + + +func shrVW(z, x []Word, s Word) (c Word) { return shrVW_g(z, x, s) } +func shrVW_g(z, x []Word, s Word) (c Word) { + if n := len(z); n > 0 { + ŝ := _W - s + w1 := x[0] + c = w1 << ŝ + for i := 0; i < n-1; i++ { + w := w1 + w1 = x[i+1] + z[i] = w>>s | w1<<ŝ + } + z[n-1] = w1 >> s + } + return +} + + +func mulAddVWW(z, x []Word, y, r Word) (c Word) { return mulAddVWW_g(z, x, y, r) } +func mulAddVWW_g(z, x []Word, y, r Word) (c Word) { + c = r + for i := range z { + c, z[i] = mulAddWWW_g(x[i], y, c) + } + return +} + + +func addMulVVW(z, x []Word, y Word) (c Word) { return addMulVVW_g(z, x, y) } +func addMulVVW_g(z, x []Word, y Word) (c Word) { + for i := range z { + z1, z0 := mulAddWWW_g(x[i], y, z[i]) + c, z[i] = addWW_g(z0, c, 0) + c += z1 + } + return +} + + +func divWVW(z []Word, xn Word, x []Word, y Word) (r Word) { return divWVW_g(z, xn, x, y) } +func divWVW_g(z []Word, xn Word, x []Word, y Word) (r Word) { + r = xn + for i := len(z) - 1; i >= 0; i-- { + z[i], r = divWW_g(r, x[i], y) + } + return +} diff --git a/libgo/go/big/arith_decl.go b/libgo/go/big/arith_decl.go new file mode 100644 index 000000000..c456d5f67 --- /dev/null +++ b/libgo/go/big/arith_decl.go @@ -0,0 +1,18 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package big + +// implemented in arith_$GOARCH.s +func mulWW(x, y Word) (z1, z0 Word) +func divWW(x1, x0, y Word) (q, r Word) +func addVV(z, x, y []Word) (c Word) +func subVV(z, x, y []Word) (c Word) +func addVW(z, x []Word, y Word) (c Word) +func subVW(z, x []Word, y Word) (c Word) +func shlVW(z, x []Word, s Word) (c Word) +func shrVW(z, x []Word, s Word) (c Word) +func mulAddVWW(z, x []Word, y, r Word) (c Word) +func addMulVVW(z, x []Word, y Word) (c Word) +func divWVW(z []Word, xn Word, x []Word, y Word) (r Word) diff --git a/libgo/go/big/arith_test.go b/libgo/go/big/arith_test.go new file mode 100644 index 000000000..934b302df --- /dev/null +++ b/libgo/go/big/arith_test.go @@ -0,0 +1,342 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package big + +import "testing" + + +type funWW func(x, y, c Word) (z1, z0 Word) +type argWW struct { + x, y, c, z1, z0 Word +} + +var sumWW = []argWW{ + {0, 0, 0, 0, 0}, + {0, 1, 0, 0, 1}, + {0, 0, 1, 0, 1}, + {0, 1, 1, 0, 2}, + {12345, 67890, 0, 0, 80235}, + {12345, 67890, 1, 0, 80236}, + {_M, 1, 0, 1, 0}, + {_M, 0, 1, 1, 0}, + {_M, 1, 1, 1, 1}, + {_M, _M, 0, 1, _M - 1}, + {_M, _M, 1, 1, _M}, +} + + +func testFunWW(t *testing.T, msg string, f funWW, a argWW) { + z1, z0 := f(a.x, a.y, a.c) + if z1 != a.z1 || z0 != a.z0 { + t.Errorf("%s%+v\n\tgot z1:z0 = %#x:%#x; want %#x:%#x", msg, a, z1, z0, a.z1, a.z0) + } +} + + +func TestFunWW(t *testing.T) { + for _, a := range sumWW { + arg := a + testFunWW(t, "addWW_g", addWW_g, arg) + + arg = argWW{a.y, a.x, a.c, a.z1, a.z0} + testFunWW(t, "addWW_g symmetric", addWW_g, arg) + + arg = argWW{a.z0, a.x, a.c, a.z1, a.y} + testFunWW(t, "subWW_g", subWW_g, arg) + + arg = argWW{a.z0, a.y, a.c, a.z1, a.x} + testFunWW(t, "subWW_g symmetric", subWW_g, arg) + } +} + + +type funVV func(z, x, y []Word) (c Word) +type argVV struct { + z, x, y nat + c Word +} + +var sumVV = []argVV{ + {}, + {nat{0}, nat{0}, nat{0}, 0}, + {nat{1}, nat{1}, nat{0}, 0}, + {nat{0}, nat{_M}, nat{1}, 1}, + {nat{80235}, nat{12345}, nat{67890}, 0}, + {nat{_M - 1}, nat{_M}, nat{_M}, 1}, + {nat{0, 0, 0, 0}, nat{_M, _M, _M, _M}, nat{1, 0, 0, 0}, 1}, + {nat{0, 0, 0, _M}, nat{_M, _M, _M, _M - 1}, nat{1, 0, 0, 0}, 0}, + {nat{0, 0, 0, 0}, nat{_M, 0, _M, 0}, nat{1, _M, 0, _M}, 1}, +} + + +func testFunVV(t *testing.T, msg string, f funVV, a argVV) { + z := make(nat, len(a.z)) + c := f(z, a.x, a.y) + for i, zi := range z { + if zi != a.z[i] { + t.Errorf("%s%+v\n\tgot z[%d] = %#x; want %#x", msg, a, i, zi, a.z[i]) + break + } + } + if c != a.c { + t.Errorf("%s%+v\n\tgot c = %#x; want %#x", msg, a, c, a.c) + } +} + + +func TestFunVV(t *testing.T) { + for _, a := range sumVV { + arg := a + testFunVV(t, "addVV_g", addVV_g, arg) + testFunVV(t, "addVV", addVV, arg) + + arg = argVV{a.z, a.y, a.x, a.c} + testFunVV(t, "addVV_g symmetric", addVV_g, arg) + testFunVV(t, "addVV symmetric", addVV, arg) + + arg = argVV{a.x, a.z, a.y, a.c} + testFunVV(t, "subVV_g", subVV_g, arg) + testFunVV(t, "subVV", subVV, arg) + + arg = argVV{a.y, a.z, a.x, a.c} + testFunVV(t, "subVV_g symmetric", subVV_g, arg) + testFunVV(t, "subVV symmetric", subVV, arg) + } +} + + +type funVW func(z, x []Word, y Word) (c Word) +type argVW struct { + z, x nat + y Word + c Word +} + +var sumVW = []argVW{ + {}, + {nil, nil, 2, 2}, + {nat{0}, nat{0}, 0, 0}, + {nat{1}, nat{0}, 1, 0}, + {nat{1}, nat{1}, 0, 0}, + {nat{0}, nat{_M}, 1, 1}, + {nat{0, 0, 0, 0}, nat{_M, _M, _M, _M}, 1, 1}, +} + +var prodVW = []argVW{ + {}, + {nat{0}, nat{0}, 0, 0}, + {nat{0}, nat{_M}, 0, 0}, + {nat{0}, nat{0}, _M, 0}, + {nat{1}, nat{1}, 1, 0}, + {nat{22793}, nat{991}, 23, 0}, + {nat{0, 0, 0, 22793}, nat{0, 0, 0, 991}, 23, 0}, + {nat{0, 0, 0, 0}, nat{7893475, 7395495, 798547395, 68943}, 0, 0}, + {nat{0, 0, 0, 0}, nat{0, 0, 0, 0}, 894375984, 0}, + {nat{_M << 1 & _M}, nat{_M}, 1 << 1, _M >> (_W - 1)}, + {nat{_M << 7 & _M}, nat{_M}, 1 << 7, _M >> (_W - 7)}, + {nat{_M << 7 & _M, _M, _M, _M}, nat{_M, _M, _M, _M}, 1 << 7, _M >> (_W - 7)}, +} + +var lshVW = []argVW{ + {}, + {nat{0}, nat{0}, 0, 0}, + {nat{0}, nat{0}, 1, 0}, + {nat{0}, nat{0}, 20, 0}, + + {nat{_M}, nat{_M}, 0, 0}, + {nat{_M << 1 & _M}, nat{_M}, 1, 1}, + {nat{_M << 20 & _M}, nat{_M}, 20, _M >> (_W - 20)}, + + {nat{_M, _M, _M}, nat{_M, _M, _M}, 0, 0}, + {nat{_M << 1 & _M, _M, _M}, nat{_M, _M, _M}, 1, 1}, + {nat{_M << 20 & _M, _M, _M}, nat{_M, _M, _M}, 20, _M >> (_W - 20)}, +} + +var rshVW = []argVW{ + {}, + {nat{0}, nat{0}, 0, 0}, + {nat{0}, nat{0}, 1, 0}, + {nat{0}, nat{0}, 20, 0}, + + {nat{_M}, nat{_M}, 0, 0}, + {nat{_M >> 1}, nat{_M}, 1, _M << (_W - 1) & _M}, + {nat{_M >> 20}, nat{_M}, 20, _M << (_W - 20) & _M}, + + {nat{_M, _M, _M}, nat{_M, _M, _M}, 0, 0}, + {nat{_M, _M, _M >> 1}, nat{_M, _M, _M}, 1, _M << (_W - 1) & _M}, + {nat{_M, _M, _M >> 20}, nat{_M, _M, _M}, 20, _M << (_W - 20) & _M}, +} + + +func testFunVW(t *testing.T, msg string, f funVW, a argVW) { + z := make(nat, len(a.z)) + c := f(z, a.x, a.y) + for i, zi := range z { + if zi != a.z[i] { + t.Errorf("%s%+v\n\tgot z[%d] = %#x; want %#x", msg, a, i, zi, a.z[i]) + break + } + } + if c != a.c { + t.Errorf("%s%+v\n\tgot c = %#x; want %#x", msg, a, c, a.c) + } +} + + +func TestFunVW(t *testing.T) { + for _, a := range sumVW { + arg := a + testFunVW(t, "addVW_g", addVW_g, arg) + testFunVW(t, "addVW", addVW, arg) + + arg = argVW{a.x, a.z, a.y, a.c} + testFunVW(t, "subVW_g", subVW_g, arg) + testFunVW(t, "subVW", subVW, arg) + } + + for _, a := range lshVW { + arg := a + testFunVW(t, "shlVW_g", shlVW_g, arg) + testFunVW(t, "shlVW", shlVW, arg) + } + + for _, a := range rshVW { + arg := a + testFunVW(t, "shrVW_g", shrVW_g, arg) + testFunVW(t, "shrVW", shrVW, arg) + } +} + + +type funVWW func(z, x []Word, y, r Word) (c Word) +type argVWW struct { + z, x nat + y, r Word + c Word +} + +var prodVWW = []argVWW{ + {}, + {nat{0}, nat{0}, 0, 0, 0}, + {nat{991}, nat{0}, 0, 991, 0}, + {nat{0}, nat{_M}, 0, 0, 0}, + {nat{991}, nat{_M}, 0, 991, 0}, + {nat{0}, nat{0}, _M, 0, 0}, + {nat{991}, nat{0}, _M, 991, 0}, + {nat{1}, nat{1}, 1, 0, 0}, + {nat{992}, nat{1}, 1, 991, 0}, + {nat{22793}, nat{991}, 23, 0, 0}, + {nat{22800}, nat{991}, 23, 7, 0}, + {nat{0, 0, 0, 22793}, nat{0, 0, 0, 991}, 23, 0, 0}, + {nat{7, 0, 0, 22793}, nat{0, 0, 0, 991}, 23, 7, 0}, + {nat{0, 0, 0, 0}, nat{7893475, 7395495, 798547395, 68943}, 0, 0, 0}, + {nat{991, 0, 0, 0}, nat{7893475, 7395495, 798547395, 68943}, 0, 991, 0}, + {nat{0, 0, 0, 0}, nat{0, 0, 0, 0}, 894375984, 0, 0}, + {nat{991, 0, 0, 0}, nat{0, 0, 0, 0}, 894375984, 991, 0}, + {nat{_M << 1 & _M}, nat{_M}, 1 << 1, 0, _M >> (_W - 1)}, + {nat{_M<<1&_M + 1}, nat{_M}, 1 << 1, 1, _M >> (_W - 1)}, + {nat{_M << 7 & _M}, nat{_M}, 1 << 7, 0, _M >> (_W - 7)}, + {nat{_M<<7&_M + 1<<6}, nat{_M}, 1 << 7, 1 << 6, _M >> (_W - 7)}, + {nat{_M << 7 & _M, _M, _M, _M}, nat{_M, _M, _M, _M}, 1 << 7, 0, _M >> (_W - 7)}, + {nat{_M<<7&_M + 1<<6, _M, _M, _M}, nat{_M, _M, _M, _M}, 1 << 7, 1 << 6, _M >> (_W - 7)}, +} + + +func testFunVWW(t *testing.T, msg string, f funVWW, a argVWW) { + z := make(nat, len(a.z)) + c := f(z, a.x, a.y, a.r) + for i, zi := range z { + if zi != a.z[i] { + t.Errorf("%s%+v\n\tgot z[%d] = %#x; want %#x", msg, a, i, zi, a.z[i]) + break + } + } + if c != a.c { + t.Errorf("%s%+v\n\tgot c = %#x; want %#x", msg, a, c, a.c) + } +} + + +// TODO(gri) mulAddVWW and divWVW are symmetric operations but +// their signature is not symmetric. Try to unify. + +type funWVW func(z []Word, xn Word, x []Word, y Word) (r Word) +type argWVW struct { + z nat + xn Word + x nat + y Word + r Word +} + +func testFunWVW(t *testing.T, msg string, f funWVW, a argWVW) { + z := make(nat, len(a.z)) + r := f(z, a.xn, a.x, a.y) + for i, zi := range z { + if zi != a.z[i] { + t.Errorf("%s%+v\n\tgot z[%d] = %#x; want %#x", msg, a, i, zi, a.z[i]) + break + } + } + if r != a.r { + t.Errorf("%s%+v\n\tgot r = %#x; want %#x", msg, a, r, a.r) + } +} + + +func TestFunVWW(t *testing.T) { + for _, a := range prodVWW { + arg := a + testFunVWW(t, "mulAddVWW_g", mulAddVWW_g, arg) + testFunVWW(t, "mulAddVWW", mulAddVWW, arg) + + if a.y != 0 && a.r < a.y { + arg := argWVW{a.x, a.c, a.z, a.y, a.r} + testFunWVW(t, "divWVW_g", divWVW_g, arg) + testFunWVW(t, "divWVW", divWVW, arg) + } + } +} + + +var mulWWTests = []struct { + x, y Word + q, r Word +}{ + {_M, _M, _M - 1, 1}, + // 32 bit only: {0xc47dfa8c, 50911, 0x98a4, 0x998587f4}, +} + + +func TestMulWW(t *testing.T) { + for i, test := range mulWWTests { + q, r := mulWW_g(test.x, test.y) + if q != test.q || r != test.r { + t.Errorf("#%d got (%x, %x) want (%x, %x)", i, q, r, test.q, test.r) + } + } +} + + +var mulAddWWWTests = []struct { + x, y, c Word + q, r Word +}{ + // TODO(agl): These will only work on 64-bit platforms. + // {15064310297182388543, 0xe7df04d2d35d5d80, 13537600649892366549, 13644450054494335067, 10832252001440893781}, + // {15064310297182388543, 0xdab2f18048baa68d, 13644450054494335067, 12869334219691522700, 14233854684711418382}, + {_M, _M, 0, _M - 1, 1}, + {_M, _M, _M, _M, 0}, +} + + +func TestMulAddWWW(t *testing.T) { + for i, test := range mulAddWWWTests { + q, r := mulAddWWW_g(test.x, test.y, test.c) + if q != test.q || r != test.r { + t.Errorf("#%d got (%x, %x) want (%x, %x)", i, q, r, test.q, test.r) + } + } +} diff --git a/libgo/go/big/calibrate_test.go b/libgo/go/big/calibrate_test.go new file mode 100644 index 000000000..c6cd2e693 --- /dev/null +++ b/libgo/go/big/calibrate_test.go @@ -0,0 +1,92 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file prints execution times for the Mul benchmark +// given different Karatsuba thresholds. The result may be +// used to manually fine-tune the threshold constant. The +// results are somewhat fragile; use repeated runs to get +// a clear picture. + +// Usage: gotest -calibrate + +package big + +import ( + "flag" + "fmt" + "testing" + "time" +) + + +var calibrate = flag.Bool("calibrate", false, "run calibration test") + + +// measure returns the time to run f +func measure(f func()) int64 { + const N = 100 + start := time.Nanoseconds() + for i := N; i > 0; i-- { + f() + } + stop := time.Nanoseconds() + return (stop - start) / N +} + + +func computeThresholds() { + fmt.Printf("Multiplication times for varying Karatsuba thresholds\n") + fmt.Printf("(run repeatedly for good results)\n") + + // determine Tk, the work load execution time using basic multiplication + karatsubaThreshold = 1e9 // disable karatsuba + Tb := measure(benchmarkMulLoad) + fmt.Printf("Tb = %dns\n", Tb) + + // thresholds + n := 8 // any lower values for the threshold lead to very slow multiplies + th1 := -1 + th2 := -1 + + var deltaOld int64 + for count := -1; count != 0; count-- { + // determine Tk, the work load execution time using Karatsuba multiplication + karatsubaThreshold = n // enable karatsuba + Tk := measure(benchmarkMulLoad) + + // improvement over Tb + delta := (Tb - Tk) * 100 / Tb + + fmt.Printf("n = %3d Tk = %8dns %4d%%", n, Tk, delta) + + // determine break-even point + if Tk < Tb && th1 < 0 { + th1 = n + fmt.Print(" break-even point") + } + + // determine diminishing return + if 0 < delta && delta < deltaOld && th2 < 0 { + th2 = n + fmt.Print(" diminishing return") + } + deltaOld = delta + + fmt.Println() + + // trigger counter + if th1 >= 0 && th2 >= 0 && count < 0 { + count = 20 // this many extra measurements after we got both thresholds + } + + n++ + } +} + + +func TestCalibrate(t *testing.T) { + if *calibrate { + computeThresholds() + } +} diff --git a/libgo/go/big/hilbert_test.go b/libgo/go/big/hilbert_test.go new file mode 100644 index 000000000..66a21214d --- /dev/null +++ b/libgo/go/big/hilbert_test.go @@ -0,0 +1,173 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// A little test program and benchmark for rational arithmetics. +// Computes a Hilbert matrix, its inverse, multiplies them +// and verifies that the product is the identity matrix. + +package big + +import ( + "fmt" + "testing" +) + + +type matrix struct { + n, m int + a []*Rat +} + + +func (a *matrix) at(i, j int) *Rat { + if !(0 <= i && i < a.n && 0 <= j && j < a.m) { + panic("index out of range") + } + return a.a[i*a.m+j] +} + + +func (a *matrix) set(i, j int, x *Rat) { + if !(0 <= i && i < a.n && 0 <= j && j < a.m) { + panic("index out of range") + } + a.a[i*a.m+j] = x +} + + +func newMatrix(n, m int) *matrix { + if !(0 <= n && 0 <= m) { + panic("illegal matrix") + } + a := new(matrix) + a.n = n + a.m = m + a.a = make([]*Rat, n*m) + return a +} + + +func newUnit(n int) *matrix { + a := newMatrix(n, n) + for i := 0; i < n; i++ { + for j := 0; j < n; j++ { + x := NewRat(0, 1) + if i == j { + x.SetInt64(1) + } + a.set(i, j, x) + } + } + return a +} + + +func newHilbert(n int) *matrix { + a := newMatrix(n, n) + for i := 0; i < n; i++ { + for j := 0; j < n; j++ { + a.set(i, j, NewRat(1, int64(i+j+1))) + } + } + return a +} + + +func newInverseHilbert(n int) *matrix { + a := newMatrix(n, n) + for i := 0; i < n; i++ { + for j := 0; j < n; j++ { + x1 := new(Rat).SetInt64(int64(i + j + 1)) + x2 := new(Rat).SetInt(new(Int).Binomial(int64(n+i), int64(n-j-1))) + x3 := new(Rat).SetInt(new(Int).Binomial(int64(n+j), int64(n-i-1))) + x4 := new(Rat).SetInt(new(Int).Binomial(int64(i+j), int64(i))) + + x1.Mul(x1, x2) + x1.Mul(x1, x3) + x1.Mul(x1, x4) + x1.Mul(x1, x4) + + if (i+j)&1 != 0 { + x1.Neg(x1) + } + + a.set(i, j, x1) + } + } + return a +} + + +func (a *matrix) mul(b *matrix) *matrix { + if a.m != b.n { + panic("illegal matrix multiply") + } + c := newMatrix(a.n, b.m) + for i := 0; i < c.n; i++ { + for j := 0; j < c.m; j++ { + x := NewRat(0, 1) + for k := 0; k < a.m; k++ { + x.Add(x, new(Rat).Mul(a.at(i, k), b.at(k, j))) + } + c.set(i, j, x) + } + } + return c +} + + +func (a *matrix) eql(b *matrix) bool { + if a.n != b.n || a.m != b.m { + return false + } + for i := 0; i < a.n; i++ { + for j := 0; j < a.m; j++ { + if a.at(i, j).Cmp(b.at(i, j)) != 0 { + return false + } + } + } + return true +} + + +func (a *matrix) String() string { + s := "" + for i := 0; i < a.n; i++ { + for j := 0; j < a.m; j++ { + s += fmt.Sprintf("\t%s", a.at(i, j)) + } + s += "\n" + } + return s +} + + +func doHilbert(t *testing.T, n int) { + a := newHilbert(n) + b := newInverseHilbert(n) + I := newUnit(n) + ab := a.mul(b) + if !ab.eql(I) { + if t == nil { + panic("Hilbert failed") + } + t.Errorf("a = %s\n", a) + t.Errorf("b = %s\n", b) + t.Errorf("a*b = %s\n", ab) + t.Errorf("I = %s\n", I) + } +} + + +func TestHilbert(t *testing.T) { + doHilbert(t, 10) +} + + +func BenchmarkHilbert(b *testing.B) { + for i := 0; i < b.N; i++ { + doHilbert(nil, 10) + } +} diff --git a/libgo/go/big/int.go b/libgo/go/big/int.go new file mode 100644 index 000000000..46e008734 --- /dev/null +++ b/libgo/go/big/int.go @@ -0,0 +1,741 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements signed multi-precision integers. + +package big + +import ( + "fmt" + "rand" +) + +// An Int represents a signed multi-precision integer. +// The zero value for an Int represents the value 0. +type Int struct { + neg bool // sign + abs nat // absolute value of the integer +} + + +var intOne = &Int{false, natOne} + + +// Sign returns: +// +// -1 if x < 0 +// 0 if x == 0 +// +1 if x > 0 +// +func (x *Int) Sign() int { + if len(x.abs) == 0 { + return 0 + } + if x.neg { + return -1 + } + return 1 +} + + +// SetInt64 sets z to x and returns z. +func (z *Int) SetInt64(x int64) *Int { + neg := false + if x < 0 { + neg = true + x = -x + } + z.abs = z.abs.setUint64(uint64(x)) + z.neg = neg + return z +} + + +// NewInt allocates and returns a new Int set to x. +func NewInt(x int64) *Int { + return new(Int).SetInt64(x) +} + + +// Set sets z to x and returns z. +func (z *Int) Set(x *Int) *Int { + z.abs = z.abs.set(x.abs) + z.neg = x.neg + return z +} + + +// Abs sets z to |x| (the absolute value of x) and returns z. +func (z *Int) Abs(x *Int) *Int { + z.abs = z.abs.set(x.abs) + z.neg = false + return z +} + + +// Neg sets z to -x and returns z. +func (z *Int) Neg(x *Int) *Int { + z.abs = z.abs.set(x.abs) + z.neg = len(z.abs) > 0 && !x.neg // 0 has no sign + return z +} + + +// Add sets z to the sum x+y and returns z. +func (z *Int) Add(x, y *Int) *Int { + neg := x.neg + if x.neg == y.neg { + // x + y == x + y + // (-x) + (-y) == -(x + y) + z.abs = z.abs.add(x.abs, y.abs) + } else { + // x + (-y) == x - y == -(y - x) + // (-x) + y == y - x == -(x - y) + if x.abs.cmp(y.abs) >= 0 { + z.abs = z.abs.sub(x.abs, y.abs) + } else { + neg = !neg + z.abs = z.abs.sub(y.abs, x.abs) + } + } + z.neg = len(z.abs) > 0 && neg // 0 has no sign + return z +} + + +// Sub sets z to the difference x-y and returns z. +func (z *Int) Sub(x, y *Int) *Int { + neg := x.neg + if x.neg != y.neg { + // x - (-y) == x + y + // (-x) - y == -(x + y) + z.abs = z.abs.add(x.abs, y.abs) + } else { + // x - y == x - y == -(y - x) + // (-x) - (-y) == y - x == -(x - y) + if x.abs.cmp(y.abs) >= 0 { + z.abs = z.abs.sub(x.abs, y.abs) + } else { + neg = !neg + z.abs = z.abs.sub(y.abs, x.abs) + } + } + z.neg = len(z.abs) > 0 && neg // 0 has no sign + return z +} + + +// Mul sets z to the product x*y and returns z. +func (z *Int) Mul(x, y *Int) *Int { + // x * y == x * y + // x * (-y) == -(x * y) + // (-x) * y == -(x * y) + // (-x) * (-y) == x * y + z.abs = z.abs.mul(x.abs, y.abs) + z.neg = len(z.abs) > 0 && x.neg != y.neg // 0 has no sign + return z +} + + +// MulRange sets z to the product of all integers +// in the range [a, b] inclusively and returns z. +// If a > b (empty range), the result is 1. +func (z *Int) MulRange(a, b int64) *Int { + switch { + case a > b: + return z.SetInt64(1) // empty range + case a <= 0 && b >= 0: + return z.SetInt64(0) // range includes 0 + } + // a <= b && (b < 0 || a > 0) + + neg := false + if a < 0 { + neg = (b-a)&1 == 0 + a, b = -b, -a + } + + z.abs = z.abs.mulRange(uint64(a), uint64(b)) + z.neg = neg + return z +} + + +// Binomial sets z to the binomial coefficient of (n, k) and returns z. +func (z *Int) Binomial(n, k int64) *Int { + var a, b Int + a.MulRange(n-k+1, n) + b.MulRange(1, k) + return z.Quo(&a, &b) +} + + +// Quo sets z to the quotient x/y for y != 0 and returns z. +// If y == 0, a division-by-zero run-time panic occurs. +// See QuoRem for more details. +func (z *Int) Quo(x, y *Int) *Int { + z.abs, _ = z.abs.div(nil, x.abs, y.abs) + z.neg = len(z.abs) > 0 && x.neg != y.neg // 0 has no sign + return z +} + + +// Rem sets z to the remainder x%y for y != 0 and returns z. +// If y == 0, a division-by-zero run-time panic occurs. +// See QuoRem for more details. +func (z *Int) Rem(x, y *Int) *Int { + _, z.abs = nat(nil).div(z.abs, x.abs, y.abs) + z.neg = len(z.abs) > 0 && x.neg // 0 has no sign + return z +} + + +// QuoRem sets z to the quotient x/y and r to the remainder x%y +// and returns the pair (z, r) for y != 0. +// If y == 0, a division-by-zero run-time panic occurs. +// +// QuoRem implements T-division and modulus (like Go): +// +// q = x/y with the result truncated to zero +// r = x - y*q +// +// (See Daan Leijen, ``Division and Modulus for Computer Scientists''.) +// +func (z *Int) QuoRem(x, y, r *Int) (*Int, *Int) { + z.abs, r.abs = z.abs.div(r.abs, x.abs, y.abs) + z.neg, r.neg = len(z.abs) > 0 && x.neg != y.neg, len(r.abs) > 0 && x.neg // 0 has no sign + return z, r +} + + +// Div sets z to the quotient x/y for y != 0 and returns z. +// If y == 0, a division-by-zero run-time panic occurs. +// See DivMod for more details. +func (z *Int) Div(x, y *Int) *Int { + y_neg := y.neg // z may be an alias for y + var r Int + z.QuoRem(x, y, &r) + if r.neg { + if y_neg { + z.Add(z, intOne) + } else { + z.Sub(z, intOne) + } + } + return z +} + + +// Mod sets z to the modulus x%y for y != 0 and returns z. +// If y == 0, a division-by-zero run-time panic occurs. +// See DivMod for more details. +func (z *Int) Mod(x, y *Int) *Int { + y0 := y // save y + if z == y || alias(z.abs, y.abs) { + y0 = new(Int).Set(y) + } + var q Int + q.QuoRem(x, y, z) + if z.neg { + if y0.neg { + z.Sub(z, y0) + } else { + z.Add(z, y0) + } + } + return z +} + + +// DivMod sets z to the quotient x div y and m to the modulus x mod y +// and returns the pair (z, m) for y != 0. +// If y == 0, a division-by-zero run-time panic occurs. +// +// DivMod implements Euclidean division and modulus (unlike Go): +// +// q = x div y such that +// m = x - y*q with 0 <= m < |q| +// +// (See Raymond T. Boute, ``The Euclidean definition of the functions +// div and mod''. ACM Transactions on Programming Languages and +// Systems (TOPLAS), 14(2):127-144, New York, NY, USA, 4/1992. +// ACM press.) +// +func (z *Int) DivMod(x, y, m *Int) (*Int, *Int) { + y0 := y // save y + if z == y || alias(z.abs, y.abs) { + y0 = new(Int).Set(y) + } + z.QuoRem(x, y, m) + if m.neg { + if y0.neg { + z.Add(z, intOne) + m.Sub(m, y0) + } else { + z.Sub(z, intOne) + m.Add(m, y0) + } + } + return z, m +} + + +// Cmp compares x and y and returns: +// +// -1 if x < y +// 0 if x == y +// +1 if x > y +// +func (x *Int) Cmp(y *Int) (r int) { + // x cmp y == x cmp y + // x cmp (-y) == x + // (-x) cmp y == y + // (-x) cmp (-y) == -(x cmp y) + switch { + case x.neg == y.neg: + r = x.abs.cmp(y.abs) + if x.neg { + r = -r + } + case x.neg: + r = -1 + default: + r = 1 + } + return +} + + +func (x *Int) String() string { + s := "" + if x.neg { + s = "-" + } + return s + x.abs.string(10) +} + + +func fmtbase(ch int) int { + switch ch { + case 'b': + return 2 + case 'o': + return 8 + case 'd': + return 10 + case 'x': + return 16 + } + return 10 +} + + +// Format is a support routine for fmt.Formatter. It accepts +// the formats 'b' (binary), 'o' (octal), 'd' (decimal) and +// 'x' (hexadecimal). +// +func (x *Int) Format(s fmt.State, ch int) { + if x.neg { + fmt.Fprint(s, "-") + } + fmt.Fprint(s, x.abs.string(fmtbase(ch))) +} + + +// Int64 returns the int64 representation of z. +// If z cannot be represented in an int64, the result is undefined. +func (x *Int) Int64() int64 { + if len(x.abs) == 0 { + return 0 + } + v := int64(x.abs[0]) + if _W == 32 && len(x.abs) > 1 { + v |= int64(x.abs[1]) << 32 + } + if x.neg { + v = -v + } + return v +} + + +// SetString sets z to the value of s, interpreted in the given base, +// and returns z and a boolean indicating success. If SetString fails, +// the value of z is undefined. +// +// If the base argument is 0, the string prefix determines the actual +// conversion base. A prefix of ``0x'' or ``0X'' selects base 16; the +// ``0'' prefix selects base 8, and a ``0b'' or ``0B'' prefix selects +// base 2. Otherwise the selected base is 10. +// +func (z *Int) SetString(s string, base int) (*Int, bool) { + if len(s) == 0 || base < 0 || base == 1 || 16 < base { + return z, false + } + + neg := s[0] == '-' + if neg || s[0] == '+' { + s = s[1:] + if len(s) == 0 { + return z, false + } + } + + var scanned int + z.abs, _, scanned = z.abs.scan(s, base) + if scanned != len(s) { + return z, false + } + z.neg = len(z.abs) > 0 && neg // 0 has no sign + + return z, true +} + + +// SetBytes interprets b as the bytes of a big-endian, unsigned integer and +// sets z to that value. +func (z *Int) SetBytes(b []byte) *Int { + const s = _S + z.abs = z.abs.make((len(b) + s - 1) / s) + + j := 0 + for len(b) >= s { + var w Word + + for i := s; i > 0; i-- { + w <<= 8 + w |= Word(b[len(b)-i]) + } + + z.abs[j] = w + j++ + b = b[0 : len(b)-s] + } + + if len(b) > 0 { + var w Word + + for i := len(b); i > 0; i-- { + w <<= 8 + w |= Word(b[len(b)-i]) + } + + z.abs[j] = w + } + + z.abs = z.abs.norm() + z.neg = false + return z +} + + +// Bytes returns the absolute value of x as a big-endian byte array. +func (z *Int) Bytes() []byte { + const s = _S + b := make([]byte, len(z.abs)*s) + + for i, w := range z.abs { + wordBytes := b[(len(z.abs)-i-1)*s : (len(z.abs)-i)*s] + for j := s - 1; j >= 0; j-- { + wordBytes[j] = byte(w) + w >>= 8 + } + } + + i := 0 + for i < len(b) && b[i] == 0 { + i++ + } + + return b[i:] +} + + +// BitLen returns the length of the absolute value of z in bits. +// The bit length of 0 is 0. +func (z *Int) BitLen() int { + return z.abs.bitLen() +} + + +// Exp sets z = x**y mod m. If m is nil, z = x**y. +// See Knuth, volume 2, section 4.6.3. +func (z *Int) Exp(x, y, m *Int) *Int { + if y.neg || len(y.abs) == 0 { + neg := x.neg + z.SetInt64(1) + z.neg = neg + return z + } + + var mWords nat + if m != nil { + mWords = m.abs + } + + z.abs = z.abs.expNN(x.abs, y.abs, mWords) + z.neg = len(z.abs) > 0 && x.neg && y.abs[0]&1 == 1 // 0 has no sign + return z +} + + +// GcdInt sets d to the greatest common divisor of a and b, which must be +// positive numbers. +// If x and y are not nil, GcdInt sets x and y such that d = a*x + b*y. +// If either a or b is not positive, GcdInt sets d = x = y = 0. +func GcdInt(d, x, y, a, b *Int) { + if a.neg || b.neg { + d.SetInt64(0) + if x != nil { + x.SetInt64(0) + } + if y != nil { + y.SetInt64(0) + } + return + } + + A := new(Int).Set(a) + B := new(Int).Set(b) + + X := new(Int) + Y := new(Int).SetInt64(1) + + lastX := new(Int).SetInt64(1) + lastY := new(Int) + + q := new(Int) + temp := new(Int) + + for len(B.abs) > 0 { + r := new(Int) + q, r = q.QuoRem(A, B, r) + + A, B = B, r + + temp.Set(X) + X.Mul(X, q) + X.neg = !X.neg + X.Add(X, lastX) + lastX.Set(temp) + + temp.Set(Y) + Y.Mul(Y, q) + Y.neg = !Y.neg + Y.Add(Y, lastY) + lastY.Set(temp) + } + + if x != nil { + *x = *lastX + } + + if y != nil { + *y = *lastY + } + + *d = *A +} + + +// ProbablyPrime performs n Miller-Rabin tests to check whether z is prime. +// If it returns true, z is prime with probability 1 - 1/4^n. +// If it returns false, z is not prime. +func ProbablyPrime(z *Int, n int) bool { + return !z.neg && z.abs.probablyPrime(n) +} + + +// Rand sets z to a pseudo-random number in [0, n) and returns z. +func (z *Int) Rand(rnd *rand.Rand, n *Int) *Int { + z.neg = false + if n.neg == true || len(n.abs) == 0 { + z.abs = nil + return z + } + z.abs = z.abs.random(rnd, n.abs, n.abs.bitLen()) + return z +} + + +// ModInverse sets z to the multiplicative inverse of g in the group ℤ/pℤ (where +// p is a prime) and returns z. +func (z *Int) ModInverse(g, p *Int) *Int { + var d Int + GcdInt(&d, z, nil, g, p) + // x and y are such that g*x + p*y = d. Since p is prime, d = 1. Taking + // that modulo p results in g*x = 1, therefore x is the inverse element. + if z.neg { + z.Add(z, p) + } + return z +} + + +// Lsh sets z = x << n and returns z. +func (z *Int) Lsh(x *Int, n uint) *Int { + z.abs = z.abs.shl(x.abs, n) + z.neg = x.neg + return z +} + + +// Rsh sets z = x >> n and returns z. +func (z *Int) Rsh(x *Int, n uint) *Int { + if x.neg { + // (-x) >> s == ^(x-1) >> s == ^((x-1) >> s) == -(((x-1) >> s) + 1) + t := z.abs.sub(x.abs, natOne) // no underflow because |x| > 0 + t = t.shr(t, n) + z.abs = t.add(t, natOne) + z.neg = true // z cannot be zero if x is negative + return z + } + + z.abs = z.abs.shr(x.abs, n) + z.neg = false + return z +} + + +// And sets z = x & y and returns z. +func (z *Int) And(x, y *Int) *Int { + if x.neg == y.neg { + if x.neg { + // (-x) & (-y) == ^(x-1) & ^(y-1) == ^((x-1) | (y-1)) == -(((x-1) | (y-1)) + 1) + x1 := nat{}.sub(x.abs, natOne) + y1 := nat{}.sub(y.abs, natOne) + z.abs = z.abs.add(z.abs.or(x1, y1), natOne) + z.neg = true // z cannot be zero if x and y are negative + return z + } + + // x & y == x & y + z.abs = z.abs.and(x.abs, y.abs) + z.neg = false + return z + } + + // x.neg != y.neg + if x.neg { + x, y = y, x // & is symmetric + } + + // x & (-y) == x & ^(y-1) == x &^ (y-1) + y1 := nat{}.sub(y.abs, natOne) + z.abs = z.abs.andNot(x.abs, y1) + z.neg = false + return z +} + + +// AndNot sets z = x &^ y and returns z. +func (z *Int) AndNot(x, y *Int) *Int { + if x.neg == y.neg { + if x.neg { + // (-x) &^ (-y) == ^(x-1) &^ ^(y-1) == ^(x-1) & (y-1) == (y-1) &^ (x-1) + x1 := nat{}.sub(x.abs, natOne) + y1 := nat{}.sub(y.abs, natOne) + z.abs = z.abs.andNot(y1, x1) + z.neg = false + return z + } + + // x &^ y == x &^ y + z.abs = z.abs.andNot(x.abs, y.abs) + z.neg = false + return z + } + + if x.neg { + // (-x) &^ y == ^(x-1) &^ y == ^(x-1) & ^y == ^((x-1) | y) == -(((x-1) | y) + 1) + x1 := nat{}.sub(x.abs, natOne) + z.abs = z.abs.add(z.abs.or(x1, y.abs), natOne) + z.neg = true // z cannot be zero if x is negative and y is positive + return z + } + + // x &^ (-y) == x &^ ^(y-1) == x & (y-1) + y1 := nat{}.add(y.abs, natOne) + z.abs = z.abs.and(x.abs, y1) + z.neg = false + return z +} + + +// Or sets z = x | y and returns z. +func (z *Int) Or(x, y *Int) *Int { + if x.neg == y.neg { + if x.neg { + // (-x) | (-y) == ^(x-1) | ^(y-1) == ^((x-1) & (y-1)) == -(((x-1) & (y-1)) + 1) + x1 := nat{}.sub(x.abs, natOne) + y1 := nat{}.sub(y.abs, natOne) + z.abs = z.abs.add(z.abs.and(x1, y1), natOne) + z.neg = true // z cannot be zero if x and y are negative + return z + } + + // x | y == x | y + z.abs = z.abs.or(x.abs, y.abs) + z.neg = false + return z + } + + // x.neg != y.neg + if x.neg { + x, y = y, x // | is symmetric + } + + // x | (-y) == x | ^(y-1) == ^((y-1) &^ x) == -(^((y-1) &^ x) + 1) + y1 := nat{}.sub(y.abs, natOne) + z.abs = z.abs.add(z.abs.andNot(y1, x.abs), natOne) + z.neg = true // z cannot be zero if one of x or y is negative + return z +} + + +// Xor sets z = x ^ y and returns z. +func (z *Int) Xor(x, y *Int) *Int { + if x.neg == y.neg { + if x.neg { + // (-x) ^ (-y) == ^(x-1) ^ ^(y-1) == (x-1) ^ (y-1) + x1 := nat{}.sub(x.abs, natOne) + y1 := nat{}.sub(y.abs, natOne) + z.abs = z.abs.xor(x1, y1) + z.neg = false + return z + } + + // x ^ y == x ^ y + z.abs = z.abs.xor(x.abs, y.abs) + z.neg = false + return z + } + + // x.neg != y.neg + if x.neg { + x, y = y, x // ^ is symmetric + } + + // x ^ (-y) == x ^ ^(y-1) == ^(x ^ (y-1)) == -((x ^ (y-1)) + 1) + y1 := nat{}.sub(y.abs, natOne) + z.abs = z.abs.add(z.abs.xor(x.abs, y1), natOne) + z.neg = true // z cannot be zero if only one of x or y is negative + return z +} + + +// Not sets z = ^x and returns z. +func (z *Int) Not(x *Int) *Int { + if x.neg { + // ^(-x) == ^(^(x-1)) == x-1 + z.abs = z.abs.sub(x.abs, natOne) + z.neg = false + return z + } + + // ^x == -x-1 == -(x+1) + z.abs = z.abs.add(x.abs, natOne) + z.neg = true // z cannot be zero if x is positive + return z +} diff --git a/libgo/go/big/int_test.go b/libgo/go/big/int_test.go new file mode 100644 index 000000000..fc981e1da --- /dev/null +++ b/libgo/go/big/int_test.go @@ -0,0 +1,1055 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package big + +import ( + "bytes" + "encoding/hex" + "fmt" + "testing" + "testing/quick" +) + + +func isNormalized(x *Int) bool { + if len(x.abs) == 0 { + return !x.neg + } + // len(x.abs) > 0 + return x.abs[len(x.abs)-1] != 0 +} + + +type funZZ func(z, x, y *Int) *Int +type argZZ struct { + z, x, y *Int +} + + +var sumZZ = []argZZ{ + {NewInt(0), NewInt(0), NewInt(0)}, + {NewInt(1), NewInt(1), NewInt(0)}, + {NewInt(1111111110), NewInt(123456789), NewInt(987654321)}, + {NewInt(-1), NewInt(-1), NewInt(0)}, + {NewInt(864197532), NewInt(-123456789), NewInt(987654321)}, + {NewInt(-1111111110), NewInt(-123456789), NewInt(-987654321)}, +} + + +var prodZZ = []argZZ{ + {NewInt(0), NewInt(0), NewInt(0)}, + {NewInt(0), NewInt(1), NewInt(0)}, + {NewInt(1), NewInt(1), NewInt(1)}, + {NewInt(-991 * 991), NewInt(991), NewInt(-991)}, + // TODO(gri) add larger products +} + + +func TestSignZ(t *testing.T) { + var zero Int + for _, a := range sumZZ { + s := a.z.Sign() + e := a.z.Cmp(&zero) + if s != e { + t.Errorf("got %d; want %d for z = %v", s, e, a.z) + } + } +} + + +func TestSetZ(t *testing.T) { + for _, a := range sumZZ { + var z Int + z.Set(a.z) + if !isNormalized(&z) { + t.Errorf("%v is not normalized", z) + } + if (&z).Cmp(a.z) != 0 { + t.Errorf("got z = %v; want %v", z, a.z) + } + } +} + + +func TestAbsZ(t *testing.T) { + var zero Int + for _, a := range sumZZ { + var z Int + z.Abs(a.z) + var e Int + e.Set(a.z) + if e.Cmp(&zero) < 0 { + e.Sub(&zero, &e) + } + if z.Cmp(&e) != 0 { + t.Errorf("got z = %v; want %v", z, e) + } + } +} + + +func testFunZZ(t *testing.T, msg string, f funZZ, a argZZ) { + var z Int + f(&z, a.x, a.y) + if !isNormalized(&z) { + t.Errorf("%s%v is not normalized", z, msg) + } + if (&z).Cmp(a.z) != 0 { + t.Errorf("%s%+v\n\tgot z = %v; want %v", msg, a, &z, a.z) + } +} + + +func TestSumZZ(t *testing.T) { + AddZZ := func(z, x, y *Int) *Int { return z.Add(x, y) } + SubZZ := func(z, x, y *Int) *Int { return z.Sub(x, y) } + for _, a := range sumZZ { + arg := a + testFunZZ(t, "AddZZ", AddZZ, arg) + + arg = argZZ{a.z, a.y, a.x} + testFunZZ(t, "AddZZ symmetric", AddZZ, arg) + + arg = argZZ{a.x, a.z, a.y} + testFunZZ(t, "SubZZ", SubZZ, arg) + + arg = argZZ{a.y, a.z, a.x} + testFunZZ(t, "SubZZ symmetric", SubZZ, arg) + } +} + + +func TestProdZZ(t *testing.T) { + MulZZ := func(z, x, y *Int) *Int { return z.Mul(x, y) } + for _, a := range prodZZ { + arg := a + testFunZZ(t, "MulZZ", MulZZ, arg) + + arg = argZZ{a.z, a.y, a.x} + testFunZZ(t, "MulZZ symmetric", MulZZ, arg) + } +} + + +// mulBytes returns x*y via grade school multiplication. Both inputs +// and the result are assumed to be in big-endian representation (to +// match the semantics of Int.Bytes and Int.SetBytes). +func mulBytes(x, y []byte) []byte { + z := make([]byte, len(x)+len(y)) + + // multiply + k0 := len(z) - 1 + for j := len(y) - 1; j >= 0; j-- { + d := int(y[j]) + if d != 0 { + k := k0 + carry := 0 + for i := len(x) - 1; i >= 0; i-- { + t := int(z[k]) + int(x[i])*d + carry + z[k], carry = byte(t), t>>8 + k-- + } + z[k] = byte(carry) + } + k0-- + } + + // normalize (remove leading 0's) + i := 0 + for i < len(z) && z[i] == 0 { + i++ + } + + return z[i:] +} + + +func checkMul(a, b []byte) bool { + var x, y, z1 Int + x.SetBytes(a) + y.SetBytes(b) + z1.Mul(&x, &y) + + var z2 Int + z2.SetBytes(mulBytes(a, b)) + + return z1.Cmp(&z2) == 0 +} + + +func TestMul(t *testing.T) { + if err := quick.Check(checkMul, nil); err != nil { + t.Error(err) + } +} + + +var mulRangesZ = []struct { + a, b int64 + prod string +}{ + // entirely positive ranges are covered by mulRangesN + {-1, 1, "0"}, + {-2, -1, "2"}, + {-3, -2, "6"}, + {-3, -1, "-6"}, + {1, 3, "6"}, + {-10, -10, "-10"}, + {0, -1, "1"}, // empty range + {-1, -100, "1"}, // empty range + {-1, 1, "0"}, // range includes 0 + {-1e9, 0, "0"}, // range includes 0 + {-1e9, 1e9, "0"}, // range includes 0 + {-10, -1, "3628800"}, // 10! + {-20, -2, "-2432902008176640000"}, // -20! + {-99, -1, + "-933262154439441526816992388562667004907159682643816214685929" + + "638952175999932299156089414639761565182862536979208272237582" + + "511852109168640000000000000000000000", // -99! + }, +} + + +func TestMulRangeZ(t *testing.T) { + var tmp Int + // test entirely positive ranges + for i, r := range mulRangesN { + prod := tmp.MulRange(int64(r.a), int64(r.b)).String() + if prod != r.prod { + t.Errorf("#%da: got %s; want %s", i, prod, r.prod) + } + } + // test other ranges + for i, r := range mulRangesZ { + prod := tmp.MulRange(r.a, r.b).String() + if prod != r.prod { + t.Errorf("#%db: got %s; want %s", i, prod, r.prod) + } + } +} + + +var stringTests = []struct { + in string + out string + base int + val int64 + ok bool +}{ + {in: "", ok: false}, + {in: "a", ok: false}, + {in: "z", ok: false}, + {in: "+", ok: false}, + {in: "-", ok: false}, + {in: "0b", ok: false}, + {in: "0x", ok: false}, + {in: "2", base: 2, ok: false}, + {in: "0b2", base: 0, ok: false}, + {in: "08", ok: false}, + {in: "8", base: 8, ok: false}, + {in: "0xg", base: 0, ok: false}, + {in: "g", base: 16, ok: false}, + {"0", "0", 0, 0, true}, + {"0", "0", 10, 0, true}, + {"0", "0", 16, 0, true}, + {"+0", "0", 0, 0, true}, + {"-0", "0", 0, 0, true}, + {"10", "10", 0, 10, true}, + {"10", "10", 10, 10, true}, + {"10", "10", 16, 16, true}, + {"-10", "-10", 16, -16, true}, + {"+10", "10", 16, 16, true}, + {"0x10", "16", 0, 16, true}, + {in: "0x10", base: 16, ok: false}, + {"-0x10", "-16", 0, -16, true}, + {"+0x10", "16", 0, 16, true}, + {"00", "0", 0, 0, true}, + {"0", "0", 8, 0, true}, + {"07", "7", 0, 7, true}, + {"7", "7", 8, 7, true}, + {"023", "19", 0, 19, true}, + {"23", "23", 8, 19, true}, + {"cafebabe", "cafebabe", 16, 0xcafebabe, true}, + {"0b0", "0", 0, 0, true}, + {"-111", "-111", 2, -7, true}, + {"-0b111", "-7", 0, -7, true}, + {"0b1001010111", "599", 0, 0x257, true}, + {"1001010111", "1001010111", 2, 0x257, true}, +} + + +func format(base int) string { + switch base { + case 2: + return "%b" + case 8: + return "%o" + case 16: + return "%x" + } + return "%d" +} + + +func TestGetString(t *testing.T) { + z := new(Int) + for i, test := range stringTests { + if !test.ok { + continue + } + z.SetInt64(test.val) + + if test.base == 10 { + s := z.String() + if s != test.out { + t.Errorf("#%da got %s; want %s", i, s, test.out) + } + } + + s := fmt.Sprintf(format(test.base), z) + if s != test.out { + t.Errorf("#%db got %s; want %s", i, s, test.out) + } + } +} + + +func TestSetString(t *testing.T) { + tmp := new(Int) + for i, test := range stringTests { + n1, ok1 := new(Int).SetString(test.in, test.base) + n2, ok2 := tmp.SetString(test.in, test.base) + expected := NewInt(test.val) + if ok1 != test.ok || ok2 != test.ok { + t.Errorf("#%d (input '%s') ok incorrect (should be %t)", i, test.in, test.ok) + continue + } + if !ok1 || !ok2 { + continue + } + + if ok1 && !isNormalized(n1) { + t.Errorf("#%d (input '%s'): %v is not normalized", i, test.in, *n1) + } + if ok2 && !isNormalized(n2) { + t.Errorf("#%d (input '%s'): %v is not normalized", i, test.in, *n2) + } + + if n1.Cmp(expected) != 0 { + t.Errorf("#%d (input '%s') got: %s want: %d", i, test.in, n1, test.val) + } + if n2.Cmp(expected) != 0 { + t.Errorf("#%d (input '%s') got: %s want: %d", i, test.in, n2, test.val) + } + } +} + + +// Examples from the Go Language Spec, section "Arithmetic operators" +var divisionSignsTests = []struct { + x, y int64 + q, r int64 // T-division + d, m int64 // Euclidian division +}{ + {5, 3, 1, 2, 1, 2}, + {-5, 3, -1, -2, -2, 1}, + {5, -3, -1, 2, -1, 2}, + {-5, -3, 1, -2, 2, 1}, + {1, 2, 0, 1, 0, 1}, + {8, 4, 2, 0, 2, 0}, +} + + +func TestDivisionSigns(t *testing.T) { + for i, test := range divisionSignsTests { + x := NewInt(test.x) + y := NewInt(test.y) + q := NewInt(test.q) + r := NewInt(test.r) + d := NewInt(test.d) + m := NewInt(test.m) + + q1 := new(Int).Quo(x, y) + r1 := new(Int).Rem(x, y) + if !isNormalized(q1) { + t.Errorf("#%d Quo: %v is not normalized", i, *q1) + } + if !isNormalized(r1) { + t.Errorf("#%d Rem: %v is not normalized", i, *r1) + } + if q1.Cmp(q) != 0 || r1.Cmp(r) != 0 { + t.Errorf("#%d QuoRem: got (%s, %s), want (%s, %s)", i, q1, r1, q, r) + } + + q2, r2 := new(Int).QuoRem(x, y, new(Int)) + if !isNormalized(q2) { + t.Errorf("#%d Quo: %v is not normalized", i, *q2) + } + if !isNormalized(r2) { + t.Errorf("#%d Rem: %v is not normalized", i, *r2) + } + if q2.Cmp(q) != 0 || r2.Cmp(r) != 0 { + t.Errorf("#%d QuoRem: got (%s, %s), want (%s, %s)", i, q2, r2, q, r) + } + + d1 := new(Int).Div(x, y) + m1 := new(Int).Mod(x, y) + if !isNormalized(d1) { + t.Errorf("#%d Div: %v is not normalized", i, *d1) + } + if !isNormalized(m1) { + t.Errorf("#%d Mod: %v is not normalized", i, *m1) + } + if d1.Cmp(d) != 0 || m1.Cmp(m) != 0 { + t.Errorf("#%d DivMod: got (%s, %s), want (%s, %s)", i, d1, m1, d, m) + } + + d2, m2 := new(Int).DivMod(x, y, new(Int)) + if !isNormalized(d2) { + t.Errorf("#%d Div: %v is not normalized", i, *d2) + } + if !isNormalized(m2) { + t.Errorf("#%d Mod: %v is not normalized", i, *m2) + } + if d2.Cmp(d) != 0 || m2.Cmp(m) != 0 { + t.Errorf("#%d DivMod: got (%s, %s), want (%s, %s)", i, d2, m2, d, m) + } + } +} + + +func checkSetBytes(b []byte) bool { + hex1 := hex.EncodeToString(new(Int).SetBytes(b).Bytes()) + hex2 := hex.EncodeToString(b) + + for len(hex1) < len(hex2) { + hex1 = "0" + hex1 + } + + for len(hex1) > len(hex2) { + hex2 = "0" + hex2 + } + + return hex1 == hex2 +} + + +func TestSetBytes(t *testing.T) { + if err := quick.Check(checkSetBytes, nil); err != nil { + t.Error(err) + } +} + + +func checkBytes(b []byte) bool { + b2 := new(Int).SetBytes(b).Bytes() + return bytes.Compare(b, b2) == 0 +} + + +func TestBytes(t *testing.T) { + if err := quick.Check(checkSetBytes, nil); err != nil { + t.Error(err) + } +} + + +func checkQuo(x, y []byte) bool { + u := new(Int).SetBytes(x) + v := new(Int).SetBytes(y) + + if len(v.abs) == 0 { + return true + } + + r := new(Int) + q, r := new(Int).QuoRem(u, v, r) + + if r.Cmp(v) >= 0 { + return false + } + + uprime := new(Int).Set(q) + uprime.Mul(uprime, v) + uprime.Add(uprime, r) + + return uprime.Cmp(u) == 0 +} + + +var quoTests = []struct { + x, y string + q, r string +}{ + { + "476217953993950760840509444250624797097991362735329973741718102894495832294430498335824897858659711275234906400899559094370964723884706254265559534144986498357", + "9353930466774385905609975137998169297361893554149986716853295022578535724979483772383667534691121982974895531435241089241440253066816724367338287092081996", + "50911", + "1", + }, + { + "11510768301994997771168", + "1328165573307167369775", + "8", + "885443715537658812968", + }, +} + + +func TestQuo(t *testing.T) { + if err := quick.Check(checkQuo, nil); err != nil { + t.Error(err) + } + + for i, test := range quoTests { + x, _ := new(Int).SetString(test.x, 10) + y, _ := new(Int).SetString(test.y, 10) + expectedQ, _ := new(Int).SetString(test.q, 10) + expectedR, _ := new(Int).SetString(test.r, 10) + + r := new(Int) + q, r := new(Int).QuoRem(x, y, r) + + if q.Cmp(expectedQ) != 0 || r.Cmp(expectedR) != 0 { + t.Errorf("#%d got (%s, %s) want (%s, %s)", i, q, r, expectedQ, expectedR) + } + } +} + + +func TestQuoStepD6(t *testing.T) { + // See Knuth, Volume 2, section 4.3.1, exercise 21. This code exercises + // a code path which only triggers 1 in 10^{-19} cases. + + u := &Int{false, nat{0, 0, 1 + 1<<(_W-1), _M ^ (1 << (_W - 1))}} + v := &Int{false, nat{5, 2 + 1<<(_W-1), 1 << (_W - 1)}} + + r := new(Int) + q, r := new(Int).QuoRem(u, v, r) + const expectedQ64 = "18446744073709551613" + const expectedR64 = "3138550867693340382088035895064302439801311770021610913807" + const expectedQ32 = "4294967293" + const expectedR32 = "39614081266355540837921718287" + if q.String() != expectedQ64 && q.String() != expectedQ32 || + r.String() != expectedR64 && r.String() != expectedR32 { + t.Errorf("got (%s, %s) want (%s, %s) or (%s, %s)", q, r, expectedQ64, expectedR64, expectedQ32, expectedR32) + } +} + + +var bitLenTests = []struct { + in string + out int +}{ + {"-1", 1}, + {"0", 0}, + {"1", 1}, + {"2", 2}, + {"4", 3}, + {"0xabc", 12}, + {"0x8000", 16}, + {"0x80000000", 32}, + {"0x800000000000", 48}, + {"0x8000000000000000", 64}, + {"0x80000000000000000000", 80}, + {"-0x4000000000000000000000", 87}, +} + + +func TestBitLen(t *testing.T) { + for i, test := range bitLenTests { + x, ok := new(Int).SetString(test.in, 0) + if !ok { + t.Errorf("#%d test input invalid: %s", i, test.in) + continue + } + + if n := x.BitLen(); n != test.out { + t.Errorf("#%d got %d want %d", i, n, test.out) + } + } +} + + +var expTests = []struct { + x, y, m string + out string +}{ + {"5", "0", "", "1"}, + {"-5", "0", "", "-1"}, + {"5", "1", "", "5"}, + {"-5", "1", "", "-5"}, + {"-2", "3", "2", "0"}, + {"5", "2", "", "25"}, + {"1", "65537", "2", "1"}, + {"0x8000000000000000", "2", "", "0x40000000000000000000000000000000"}, + {"0x8000000000000000", "2", "6719", "4944"}, + {"0x8000000000000000", "3", "6719", "5447"}, + {"0x8000000000000000", "1000", "6719", "1603"}, + {"0x8000000000000000", "1000000", "6719", "3199"}, + { + "2938462938472983472983659726349017249287491026512746239764525612965293865296239471239874193284792387498274256129746192347", + "298472983472983471903246121093472394872319615612417471234712061", + "29834729834729834729347290846729561262544958723956495615629569234729836259263598127342374289365912465901365498236492183464", + "23537740700184054162508175125554701713153216681790245129157191391322321508055833908509185839069455749219131480588829346291", + }, +} + + +func TestExp(t *testing.T) { + for i, test := range expTests { + x, ok1 := new(Int).SetString(test.x, 0) + y, ok2 := new(Int).SetString(test.y, 0) + out, ok3 := new(Int).SetString(test.out, 0) + + var ok4 bool + var m *Int + + if len(test.m) == 0 { + m, ok4 = nil, true + } else { + m, ok4 = new(Int).SetString(test.m, 0) + } + + if !ok1 || !ok2 || !ok3 || !ok4 { + t.Errorf("#%d: error in input", i) + continue + } + + z := y.Exp(x, y, m) + if !isNormalized(z) { + t.Errorf("#%d: %v is not normalized", i, *z) + } + if z.Cmp(out) != 0 { + t.Errorf("#%d: got %s want %s", i, z, out) + } + } +} + + +func checkGcd(aBytes, bBytes []byte) bool { + a := new(Int).SetBytes(aBytes) + b := new(Int).SetBytes(bBytes) + + x := new(Int) + y := new(Int) + d := new(Int) + + GcdInt(d, x, y, a, b) + x.Mul(x, a) + y.Mul(y, b) + x.Add(x, y) + + return x.Cmp(d) == 0 +} + + +var gcdTests = []struct { + a, b int64 + d, x, y int64 +}{ + {120, 23, 1, -9, 47}, +} + + +func TestGcd(t *testing.T) { + for i, test := range gcdTests { + a := NewInt(test.a) + b := NewInt(test.b) + + x := new(Int) + y := new(Int) + d := new(Int) + + expectedX := NewInt(test.x) + expectedY := NewInt(test.y) + expectedD := NewInt(test.d) + + GcdInt(d, x, y, a, b) + + if expectedX.Cmp(x) != 0 || + expectedY.Cmp(y) != 0 || + expectedD.Cmp(d) != 0 { + t.Errorf("#%d got (%s %s %s) want (%s %s %s)", i, x, y, d, expectedX, expectedY, expectedD) + } + } + + quick.Check(checkGcd, nil) +} + + +var primes = []string{ + "2", + "3", + "5", + "7", + "11", + + "13756265695458089029", + "13496181268022124907", + "10953742525620032441", + "17908251027575790097", + + // http://code.google.com/p/go/issues/detail?id=638 + "18699199384836356663", + + "98920366548084643601728869055592650835572950932266967461790948584315647051443", + "94560208308847015747498523884063394671606671904944666360068158221458669711639", + + // http://primes.utm.edu/lists/small/small3.html + "449417999055441493994709297093108513015373787049558499205492347871729927573118262811508386655998299074566974373711472560655026288668094291699357843464363003144674940345912431129144354948751003607115263071543163", + "230975859993204150666423538988557839555560243929065415434980904258310530753006723857139742334640122533598517597674807096648905501653461687601339782814316124971547968912893214002992086353183070342498989426570593", + "5521712099665906221540423207019333379125265462121169655563495403888449493493629943498064604536961775110765377745550377067893607246020694972959780839151452457728855382113555867743022746090187341871655890805971735385789993", + "203956878356401977405765866929034577280193993314348263094772646453283062722701277632936616063144088173312372882677123879538709400158306567338328279154499698366071906766440037074217117805690872792848149112022286332144876183376326512083574821647933992961249917319836219304274280243803104015000563790123", +} + + +var composites = []string{ + "21284175091214687912771199898307297748211672914763848041968395774954376176754", + "6084766654921918907427900243509372380954290099172559290432744450051395395951", + "84594350493221918389213352992032324280367711247940675652888030554255915464401", + "82793403787388584738507275144194252681", +} + + +func TestProbablyPrime(t *testing.T) { + for i, s := range primes { + p, _ := new(Int).SetString(s, 10) + if !ProbablyPrime(p, 20) { + t.Errorf("#%d prime found to be non-prime (%s)", i, s) + } + } + + for i, s := range composites { + c, _ := new(Int).SetString(s, 10) + if ProbablyPrime(c, 20) { + t.Errorf("#%d composite found to be prime (%s)", i, s) + } + } +} + + +type intShiftTest struct { + in string + shift uint + out string +} + + +var rshTests = []intShiftTest{ + {"0", 0, "0"}, + {"-0", 0, "0"}, + {"0", 1, "0"}, + {"0", 2, "0"}, + {"1", 0, "1"}, + {"1", 1, "0"}, + {"1", 2, "0"}, + {"2", 0, "2"}, + {"2", 1, "1"}, + {"-1", 0, "-1"}, + {"-1", 1, "-1"}, + {"-1", 10, "-1"}, + {"-100", 2, "-25"}, + {"-100", 3, "-13"}, + {"-100", 100, "-1"}, + {"4294967296", 0, "4294967296"}, + {"4294967296", 1, "2147483648"}, + {"4294967296", 2, "1073741824"}, + {"18446744073709551616", 0, "18446744073709551616"}, + {"18446744073709551616", 1, "9223372036854775808"}, + {"18446744073709551616", 2, "4611686018427387904"}, + {"18446744073709551616", 64, "1"}, + {"340282366920938463463374607431768211456", 64, "18446744073709551616"}, + {"340282366920938463463374607431768211456", 128, "1"}, +} + + +func TestRsh(t *testing.T) { + for i, test := range rshTests { + in, _ := new(Int).SetString(test.in, 10) + expected, _ := new(Int).SetString(test.out, 10) + out := new(Int).Rsh(in, test.shift) + + if !isNormalized(out) { + t.Errorf("#%d: %v is not normalized", i, *out) + } + if out.Cmp(expected) != 0 { + t.Errorf("#%d: got %s want %s", i, out, expected) + } + } +} + + +func TestRshSelf(t *testing.T) { + for i, test := range rshTests { + z, _ := new(Int).SetString(test.in, 10) + expected, _ := new(Int).SetString(test.out, 10) + z.Rsh(z, test.shift) + + if !isNormalized(z) { + t.Errorf("#%d: %v is not normalized", i, *z) + } + if z.Cmp(expected) != 0 { + t.Errorf("#%d: got %s want %s", i, z, expected) + } + } +} + + +var lshTests = []intShiftTest{ + {"0", 0, "0"}, + {"0", 1, "0"}, + {"0", 2, "0"}, + {"1", 0, "1"}, + {"1", 1, "2"}, + {"1", 2, "4"}, + {"2", 0, "2"}, + {"2", 1, "4"}, + {"2", 2, "8"}, + {"-87", 1, "-174"}, + {"4294967296", 0, "4294967296"}, + {"4294967296", 1, "8589934592"}, + {"4294967296", 2, "17179869184"}, + {"18446744073709551616", 0, "18446744073709551616"}, + {"9223372036854775808", 1, "18446744073709551616"}, + {"4611686018427387904", 2, "18446744073709551616"}, + {"1", 64, "18446744073709551616"}, + {"18446744073709551616", 64, "340282366920938463463374607431768211456"}, + {"1", 128, "340282366920938463463374607431768211456"}, +} + + +func TestLsh(t *testing.T) { + for i, test := range lshTests { + in, _ := new(Int).SetString(test.in, 10) + expected, _ := new(Int).SetString(test.out, 10) + out := new(Int).Lsh(in, test.shift) + + if !isNormalized(out) { + t.Errorf("#%d: %v is not normalized", i, *out) + } + if out.Cmp(expected) != 0 { + t.Errorf("#%d: got %s want %s", i, out, expected) + } + } +} + + +func TestLshSelf(t *testing.T) { + for i, test := range lshTests { + z, _ := new(Int).SetString(test.in, 10) + expected, _ := new(Int).SetString(test.out, 10) + z.Lsh(z, test.shift) + + if !isNormalized(z) { + t.Errorf("#%d: %v is not normalized", i, *z) + } + if z.Cmp(expected) != 0 { + t.Errorf("#%d: got %s want %s", i, z, expected) + } + } +} + + +func TestLshRsh(t *testing.T) { + for i, test := range rshTests { + in, _ := new(Int).SetString(test.in, 10) + out := new(Int).Lsh(in, test.shift) + out = out.Rsh(out, test.shift) + + if !isNormalized(out) { + t.Errorf("#%d: %v is not normalized", i, *out) + } + if in.Cmp(out) != 0 { + t.Errorf("#%d: got %s want %s", i, out, in) + } + } + for i, test := range lshTests { + in, _ := new(Int).SetString(test.in, 10) + out := new(Int).Lsh(in, test.shift) + out.Rsh(out, test.shift) + + if !isNormalized(out) { + t.Errorf("#%d: %v is not normalized", i, *out) + } + if in.Cmp(out) != 0 { + t.Errorf("#%d: got %s want %s", i, out, in) + } + } +} + + +var int64Tests = []int64{ + 0, + 1, + -1, + 4294967295, + -4294967295, + 4294967296, + -4294967296, + 9223372036854775807, + -9223372036854775807, + -9223372036854775808, +} + + +func TestInt64(t *testing.T) { + for i, testVal := range int64Tests { + in := NewInt(testVal) + out := in.Int64() + + if out != testVal { + t.Errorf("#%d got %d want %d", i, out, testVal) + } + } +} + + +var bitwiseTests = []struct { + x, y string + and, or, xor, andNot string +}{ + {"0x00", "0x00", "0x00", "0x00", "0x00", "0x00"}, + {"0x00", "0x01", "0x00", "0x01", "0x01", "0x00"}, + {"0x01", "0x00", "0x00", "0x01", "0x01", "0x01"}, + {"-0x01", "0x00", "0x00", "-0x01", "-0x01", "-0x01"}, + {"-0xaf", "-0x50", "-0xf0", "-0x0f", "0xe1", "0x41"}, + {"0x00", "-0x01", "0x00", "-0x01", "-0x01", "0x00"}, + {"0x01", "0x01", "0x01", "0x01", "0x00", "0x00"}, + {"-0x01", "-0x01", "-0x01", "-0x01", "0x00", "0x00"}, + {"0x07", "0x08", "0x00", "0x0f", "0x0f", "0x07"}, + {"0x05", "0x0f", "0x05", "0x0f", "0x0a", "0x00"}, + {"0x013ff6", "0x9a4e", "0x1a46", "0x01bffe", "0x01a5b8", "0x0125b0"}, + {"-0x013ff6", "0x9a4e", "0x800a", "-0x0125b2", "-0x01a5bc", "-0x01c000"}, + {"-0x013ff6", "-0x9a4e", "-0x01bffe", "-0x1a46", "0x01a5b8", "0x8008"}, + { + "0x1000009dc6e3d9822cba04129bcbe3401", + "0xb9bd7d543685789d57cb918e833af352559021483cdb05cc21fd", + "0x1000001186210100001000009048c2001", + "0xb9bd7d543685789d57cb918e8bfeff7fddb2ebe87dfbbdfe35fd", + "0xb9bd7d543685789d57ca918e8ae69d6fcdb2eae87df2b97215fc", + "0x8c40c2d8822caa04120b8321400", + }, + { + "0x1000009dc6e3d9822cba04129bcbe3401", + "-0xb9bd7d543685789d57cb918e833af352559021483cdb05cc21fd", + "0x8c40c2d8822caa04120b8321401", + "-0xb9bd7d543685789d57ca918e82229142459020483cd2014001fd", + "-0xb9bd7d543685789d57ca918e8ae69d6fcdb2eae87df2b97215fe", + "0x1000001186210100001000009048c2000", + }, + { + "-0x1000009dc6e3d9822cba04129bcbe3401", + "-0xb9bd7d543685789d57cb918e833af352559021483cdb05cc21fd", + "-0xb9bd7d543685789d57cb918e8bfeff7fddb2ebe87dfbbdfe35fd", + "-0x1000001186210100001000009048c2001", + "0xb9bd7d543685789d57ca918e8ae69d6fcdb2eae87df2b97215fc", + "0xb9bd7d543685789d57ca918e82229142459020483cd2014001fc", + }, +} + + +type bitFun func(z, x, y *Int) *Int + +func testBitFun(t *testing.T, msg string, f bitFun, x, y *Int, exp string) { + expected := new(Int) + expected.SetString(exp, 0) + + out := f(new(Int), x, y) + if out.Cmp(expected) != 0 { + t.Errorf("%s: got %s want %s", msg, out, expected) + } +} + + +func testBitFunSelf(t *testing.T, msg string, f bitFun, x, y *Int, exp string) { + self := new(Int) + self.Set(x) + expected := new(Int) + expected.SetString(exp, 0) + + self = f(self, self, y) + if self.Cmp(expected) != 0 { + t.Errorf("%s: got %s want %s", msg, self, expected) + } +} + + +func TestBitwise(t *testing.T) { + x := new(Int) + y := new(Int) + for _, test := range bitwiseTests { + x.SetString(test.x, 0) + y.SetString(test.y, 0) + + testBitFun(t, "and", (*Int).And, x, y, test.and) + testBitFunSelf(t, "and", (*Int).And, x, y, test.and) + testBitFun(t, "andNot", (*Int).AndNot, x, y, test.andNot) + testBitFunSelf(t, "andNot", (*Int).AndNot, x, y, test.andNot) + testBitFun(t, "or", (*Int).Or, x, y, test.or) + testBitFunSelf(t, "or", (*Int).Or, x, y, test.or) + testBitFun(t, "xor", (*Int).Xor, x, y, test.xor) + testBitFunSelf(t, "xor", (*Int).Xor, x, y, test.xor) + } +} + + +var notTests = []struct { + in string + out string +}{ + {"0", "-1"}, + {"1", "-2"}, + {"7", "-8"}, + {"0", "-1"}, + {"-81910", "81909"}, + { + "298472983472983471903246121093472394872319615612417471234712061", + "-298472983472983471903246121093472394872319615612417471234712062", + }, +} + +func TestNot(t *testing.T) { + in := new(Int) + out := new(Int) + expected := new(Int) + for i, test := range notTests { + in.SetString(test.in, 10) + expected.SetString(test.out, 10) + out = out.Not(in) + if out.Cmp(expected) != 0 { + t.Errorf("#%d: got %s want %s", i, out, expected) + } + out = out.Not(out) + if out.Cmp(in) != 0 { + t.Errorf("#%d: got %s want %s", i, out, in) + } + } +} + + +var modInverseTests = []struct { + element string + prime string +}{ + {"1", "7"}, + {"1", "13"}, + {"239487239847", "2410312426921032588552076022197566074856950548502459942654116941958108831682612228890093858261341614673227141477904012196503648957050582631942730706805009223062734745341073406696246014589361659774041027169249453200378729434170325843778659198143763193776859869524088940195577346119843545301547043747207749969763750084308926339295559968882457872412993810129130294592999947926365264059284647209730384947211681434464714438488520940127459844288859336526896320919633919"}, +} + +func TestModInverse(t *testing.T) { + var element, prime Int + one := NewInt(1) + for i, test := range modInverseTests { + (&element).SetString(test.element, 10) + (&prime).SetString(test.prime, 10) + inverse := new(Int).ModInverse(&element, &prime) + inverse.Mul(inverse, &element) + inverse.Mod(inverse, &prime) + if inverse.Cmp(one) != 0 { + t.Errorf("#%d: failed (e·e^(-1)=%s)", i, inverse) + } + } +} diff --git a/libgo/go/big/nat.go b/libgo/go/big/nat.go new file mode 100644 index 000000000..a308f69e8 --- /dev/null +++ b/libgo/go/big/nat.go @@ -0,0 +1,1067 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file contains operations on unsigned multi-precision integers. +// These are the building blocks for the operations on signed integers +// and rationals. + +// This package implements multi-precision arithmetic (big numbers). +// The following numeric types are supported: +// +// - Int signed integers +// - Rat rational numbers +// +// All methods on Int take the result as the receiver; if it is one +// of the operands it may be overwritten (and its memory reused). +// To enable chaining of operations, the result is also returned. +// +package big + +import "rand" + +// An unsigned integer x of the form +// +// x = x[n-1]*_B^(n-1) + x[n-2]*_B^(n-2) + ... + x[1]*_B + x[0] +// +// with 0 <= x[i] < _B and 0 <= i < n is stored in a slice of length n, +// with the digits x[i] as the slice elements. +// +// A number is normalized if the slice contains no leading 0 digits. +// During arithmetic operations, denormalized values may occur but are +// always normalized before returning the final result. The normalized +// representation of 0 is the empty or nil slice (length = 0). + +type nat []Word + +var ( + natOne = nat{1} + natTwo = nat{2} + natTen = nat{10} +) + + +func (z nat) clear() { + for i := range z { + z[i] = 0 + } +} + + +func (z nat) norm() nat { + i := len(z) + for i > 0 && z[i-1] == 0 { + i-- + } + return z[0:i] +} + + +func (z nat) make(n int) nat { + if n <= cap(z) { + return z[0:n] // reuse z + } + // Choosing a good value for e has significant performance impact + // because it increases the chance that a value can be reused. + const e = 4 // extra capacity + return make(nat, n, n+e) +} + + +func (z nat) setWord(x Word) nat { + if x == 0 { + return z.make(0) + } + z = z.make(1) + z[0] = x + return z +} + + +func (z nat) setUint64(x uint64) nat { + // single-digit values + if w := Word(x); uint64(w) == x { + return z.setWord(w) + } + + // compute number of words n required to represent x + n := 0 + for t := x; t > 0; t >>= _W { + n++ + } + + // split x into n words + z = z.make(n) + for i := range z { + z[i] = Word(x & _M) + x >>= _W + } + + return z +} + + +func (z nat) set(x nat) nat { + z = z.make(len(x)) + copy(z, x) + return z +} + + +func (z nat) add(x, y nat) nat { + m := len(x) + n := len(y) + + switch { + case m < n: + return z.add(y, x) + case m == 0: + // n == 0 because m >= n; result is 0 + return z.make(0) + case n == 0: + // result is x + return z.set(x) + } + // m > 0 + + z = z.make(m + 1) + c := addVV(z[0:n], x, y) + if m > n { + c = addVW(z[n:m], x[n:], c) + } + z[m] = c + + return z.norm() +} + + +func (z nat) sub(x, y nat) nat { + m := len(x) + n := len(y) + + switch { + case m < n: + panic("underflow") + case m == 0: + // n == 0 because m >= n; result is 0 + return z.make(0) + case n == 0: + // result is x + return z.set(x) + } + // m > 0 + + z = z.make(m) + c := subVV(z[0:n], x, y) + if m > n { + c = subVW(z[n:], x[n:], c) + } + if c != 0 { + panic("underflow") + } + + return z.norm() +} + + +func (x nat) cmp(y nat) (r int) { + m := len(x) + n := len(y) + if m != n || m == 0 { + switch { + case m < n: + r = -1 + case m > n: + r = 1 + } + return + } + + i := m - 1 + for i > 0 && x[i] == y[i] { + i-- + } + + switch { + case x[i] < y[i]: + r = -1 + case x[i] > y[i]: + r = 1 + } + return +} + + +func (z nat) mulAddWW(x nat, y, r Word) nat { + m := len(x) + if m == 0 || y == 0 { + return z.setWord(r) // result is r + } + // m > 0 + + z = z.make(m + 1) + z[m] = mulAddVWW(z[0:m], x, y, r) + + return z.norm() +} + + +// basicMul multiplies x and y and leaves the result in z. +// The (non-normalized) result is placed in z[0 : len(x) + len(y)]. +func basicMul(z, x, y nat) { + z[0 : len(x)+len(y)].clear() // initialize z + for i, d := range y { + if d != 0 { + z[len(x)+i] = addMulVVW(z[i:i+len(x)], x, d) + } + } +} + + +// Fast version of z[0:n+n>>1].add(z[0:n+n>>1], x[0:n]) w/o bounds checks. +// Factored out for readability - do not use outside karatsuba. +func karatsubaAdd(z, x nat, n int) { + if c := addVV(z[0:n], z, x); c != 0 { + addVW(z[n:n+n>>1], z[n:], c) + } +} + + +// Like karatsubaAdd, but does subtract. +func karatsubaSub(z, x nat, n int) { + if c := subVV(z[0:n], z, x); c != 0 { + subVW(z[n:n+n>>1], z[n:], c) + } +} + + +// Operands that are shorter than karatsubaThreshold are multiplied using +// "grade school" multiplication; for longer operands the Karatsuba algorithm +// is used. +var karatsubaThreshold int = 32 // computed by calibrate.go + +// karatsuba multiplies x and y and leaves the result in z. +// Both x and y must have the same length n and n must be a +// power of 2. The result vector z must have len(z) >= 6*n. +// The (non-normalized) result is placed in z[0 : 2*n]. +func karatsuba(z, x, y nat) { + n := len(y) + + // Switch to basic multiplication if numbers are odd or small. + // (n is always even if karatsubaThreshold is even, but be + // conservative) + if n&1 != 0 || n < karatsubaThreshold || n < 2 { + basicMul(z, x, y) + return + } + // n&1 == 0 && n >= karatsubaThreshold && n >= 2 + + // Karatsuba multiplication is based on the observation that + // for two numbers x and y with: + // + // x = x1*b + x0 + // y = y1*b + y0 + // + // the product x*y can be obtained with 3 products z2, z1, z0 + // instead of 4: + // + // x*y = x1*y1*b*b + (x1*y0 + x0*y1)*b + x0*y0 + // = z2*b*b + z1*b + z0 + // + // with: + // + // xd = x1 - x0 + // yd = y0 - y1 + // + // z1 = xd*yd + z1 + z0 + // = (x1-x0)*(y0 - y1) + z1 + z0 + // = x1*y0 - x1*y1 - x0*y0 + x0*y1 + z1 + z0 + // = x1*y0 - z1 - z0 + x0*y1 + z1 + z0 + // = x1*y0 + x0*y1 + + // split x, y into "digits" + n2 := n >> 1 // n2 >= 1 + x1, x0 := x[n2:], x[0:n2] // x = x1*b + y0 + y1, y0 := y[n2:], y[0:n2] // y = y1*b + y0 + + // z is used for the result and temporary storage: + // + // 6*n 5*n 4*n 3*n 2*n 1*n 0*n + // z = [z2 copy|z0 copy| xd*yd | yd:xd | x1*y1 | x0*y0 ] + // + // For each recursive call of karatsuba, an unused slice of + // z is passed in that has (at least) half the length of the + // caller's z. + + // compute z0 and z2 with the result "in place" in z + karatsuba(z, x0, y0) // z0 = x0*y0 + karatsuba(z[n:], x1, y1) // z2 = x1*y1 + + // compute xd (or the negative value if underflow occurs) + s := 1 // sign of product xd*yd + xd := z[2*n : 2*n+n2] + if subVV(xd, x1, x0) != 0 { // x1-x0 + s = -s + subVV(xd, x0, x1) // x0-x1 + } + + // compute yd (or the negative value if underflow occurs) + yd := z[2*n+n2 : 3*n] + if subVV(yd, y0, y1) != 0 { // y0-y1 + s = -s + subVV(yd, y1, y0) // y1-y0 + } + + // p = (x1-x0)*(y0-y1) == x1*y0 - x1*y1 - x0*y0 + x0*y1 for s > 0 + // p = (x0-x1)*(y0-y1) == x0*y0 - x0*y1 - x1*y0 + x1*y1 for s < 0 + p := z[n*3:] + karatsuba(p, xd, yd) + + // save original z2:z0 + // (ok to use upper half of z since we're done recursing) + r := z[n*4:] + copy(r, z) + + // add up all partial products + // + // 2*n n 0 + // z = [ z2 | z0 ] + // + [ z0 ] + // + [ z2 ] + // + [ p ] + // + karatsubaAdd(z[n2:], r, n) + karatsubaAdd(z[n2:], r[n:], n) + if s > 0 { + karatsubaAdd(z[n2:], p, n) + } else { + karatsubaSub(z[n2:], p, n) + } +} + + +// alias returns true if x and y share the same base array. +func alias(x, y nat) bool { + return cap(x) > 0 && cap(y) > 0 && &x[0:cap(x)][cap(x)-1] == &y[0:cap(y)][cap(y)-1] +} + + +// addAt implements z += x*(1<<(_W*i)); z must be long enough. +// (we don't use nat.add because we need z to stay the same +// slice, and we don't need to normalize z after each addition) +func addAt(z, x nat, i int) { + if n := len(x); n > 0 { + if c := addVV(z[i:i+n], z[i:], x); c != 0 { + j := i + n + if j < len(z) { + addVW(z[j:], z[j:], c) + } + } + } +} + + +func max(x, y int) int { + if x > y { + return x + } + return y +} + + +// karatsubaLen computes an approximation to the maximum k <= n such that +// k = p<= 0. Thus, the +// result is the largest number that can be divided repeatedly by 2 before +// becoming about the value of karatsubaThreshold. +func karatsubaLen(n int) int { + i := uint(0) + for n > karatsubaThreshold { + n >>= 1 + i++ + } + return n << i +} + + +func (z nat) mul(x, y nat) nat { + m := len(x) + n := len(y) + + switch { + case m < n: + return z.mul(y, x) + case m == 0 || n == 0: + return z.make(0) + case n == 1: + return z.mulAddWW(x, y[0], 0) + } + // m >= n > 1 + + // determine if z can be reused + if alias(z, x) || alias(z, y) { + z = nil // z is an alias for x or y - cannot reuse + } + + // use basic multiplication if the numbers are small + if n < karatsubaThreshold || n < 2 { + z = z.make(m + n) + basicMul(z, x, y) + return z.norm() + } + // m >= n && n >= karatsubaThreshold && n >= 2 + + // determine Karatsuba length k such that + // + // x = x1*b + x0 + // y = y1*b + y0 (and k <= len(y), which implies k <= len(x)) + // b = 1<<(_W*k) ("base" of digits xi, yi) + // + k := karatsubaLen(n) + // k <= n + + // multiply x0 and y0 via Karatsuba + x0 := x[0:k] // x0 is not normalized + y0 := y[0:k] // y0 is not normalized + z = z.make(max(6*k, m+n)) // enough space for karatsuba of x0*y0 and full result of x*y + karatsuba(z, x0, y0) + z = z[0 : m+n] // z has final length but may be incomplete, upper portion is garbage + + // If x1 and/or y1 are not 0, add missing terms to z explicitly: + // + // m+n 2*k 0 + // z = [ ... | x0*y0 ] + // + [ x1*y1 ] + // + [ x1*y0 ] + // + [ x0*y1 ] + // + if k < n || m != n { + x1 := x[k:] // x1 is normalized because x is + y1 := y[k:] // y1 is normalized because y is + var t nat + t = t.mul(x1, y1) + copy(z[2*k:], t) + z[2*k+len(t):].clear() // upper portion of z is garbage + t = t.mul(x1, y0.norm()) + addAt(z, t, k) + t = t.mul(x0.norm(), y1) + addAt(z, t, k) + } + + return z.norm() +} + + +// mulRange computes the product of all the unsigned integers in the +// range [a, b] inclusively. If a > b (empty range), the result is 1. +func (z nat) mulRange(a, b uint64) nat { + switch { + case a == 0: + // cut long ranges short (optimization) + return z.setUint64(0) + case a > b: + return z.setUint64(1) + case a == b: + return z.setUint64(a) + case a+1 == b: + return z.mul(nat(nil).setUint64(a), nat(nil).setUint64(b)) + } + m := (a + b) / 2 + return z.mul(nat(nil).mulRange(a, m), nat(nil).mulRange(m+1, b)) +} + + +// q = (x-r)/y, with 0 <= r < y +func (z nat) divW(x nat, y Word) (q nat, r Word) { + m := len(x) + switch { + case y == 0: + panic("division by zero") + case y == 1: + q = z.set(x) // result is x + return + case m == 0: + q = z.make(0) // result is 0 + return + } + // m > 0 + z = z.make(m) + r = divWVW(z, 0, x, y) + q = z.norm() + return +} + + +func (z nat) div(z2, u, v nat) (q, r nat) { + if len(v) == 0 { + panic("division by zero") + } + + if u.cmp(v) < 0 { + q = z.make(0) + r = z2.set(u) + return + } + + if len(v) == 1 { + var rprime Word + q, rprime = z.divW(u, v[0]) + if rprime > 0 { + r = z2.make(1) + r[0] = rprime + } else { + r = z2.make(0) + } + return + } + + q, r = z.divLarge(z2, u, v) + return +} + + +// q = (uIn-r)/v, with 0 <= r < y +// Uses z as storage for q, and u as storage for r if possible. +// See Knuth, Volume 2, section 4.3.1, Algorithm D. +// Preconditions: +// len(v) >= 2 +// len(uIn) >= len(v) +func (z nat) divLarge(u, uIn, v nat) (q, r nat) { + n := len(v) + m := len(uIn) - n + + // determine if z can be reused + // TODO(gri) should find a better solution - this if statement + // is very costly (see e.g. time pidigits -s -n 10000) + if alias(z, uIn) || alias(z, v) { + z = nil // z is an alias for uIn or v - cannot reuse + } + q = z.make(m + 1) + + qhatv := make(nat, n+1) + if alias(u, uIn) || alias(u, v) { + u = nil // u is an alias for uIn or v - cannot reuse + } + u = u.make(len(uIn) + 1) + u.clear() + + // D1. + shift := Word(leadingZeros(v[n-1])) + shlVW(v, v, shift) + u[len(uIn)] = shlVW(u[0:len(uIn)], uIn, shift) + + // D2. + for j := m; j >= 0; j-- { + // D3. + qhat := Word(_M) + if u[j+n] != v[n-1] { + var rhat Word + qhat, rhat = divWW(u[j+n], u[j+n-1], v[n-1]) + + // x1 | x2 = q̂v_{n-2} + x1, x2 := mulWW(qhat, v[n-2]) + // test if q̂v_{n-2} > br̂ + u_{j+n-2} + for greaterThan(x1, x2, rhat, u[j+n-2]) { + qhat-- + prevRhat := rhat + rhat += v[n-1] + // v[n-1] >= 0, so this tests for overflow. + if rhat < prevRhat { + break + } + x1, x2 = mulWW(qhat, v[n-2]) + } + } + + // D4. + qhatv[n] = mulAddVWW(qhatv[0:n], v, qhat, 0) + + c := subVV(u[j:j+len(qhatv)], u[j:], qhatv) + if c != 0 { + c := addVV(u[j:j+n], u[j:], v) + u[j+n] += c + qhat-- + } + + q[j] = qhat + } + + q = q.norm() + shrVW(u, u, shift) + shrVW(v, v, shift) + r = u.norm() + + return q, r +} + + +// Length of x in bits. x must be normalized. +func (x nat) bitLen() int { + if i := len(x) - 1; i >= 0 { + return i*_W + bitLen(x[i]) + } + return 0 +} + + +func hexValue(ch byte) int { + var d byte + switch { + case '0' <= ch && ch <= '9': + d = ch - '0' + case 'a' <= ch && ch <= 'f': + d = ch - 'a' + 10 + case 'A' <= ch && ch <= 'F': + d = ch - 'A' + 10 + default: + return -1 + } + return int(d) +} + + +// scan returns the natural number corresponding to the +// longest possible prefix of s representing a natural number in a +// given conversion base, the actual conversion base used, and the +// prefix length. The syntax of natural numbers follows the syntax +// of unsigned integer literals in Go. +// +// If the base argument is 0, the string prefix determines the actual +// conversion base. A prefix of ``0x'' or ``0X'' selects base 16; the +// ``0'' prefix selects base 8, and a ``0b'' or ``0B'' prefix selects +// base 2. Otherwise the selected base is 10. +// +func (z nat) scan(s string, base int) (nat, int, int) { + // determine base if necessary + i, n := 0, len(s) + if base == 0 { + base = 10 + if n > 0 && s[0] == '0' { + base, i = 8, 1 + if n > 1 { + switch s[1] { + case 'x', 'X': + base, i = 16, 2 + case 'b', 'B': + base, i = 2, 2 + } + } + } + } + + // reject illegal bases or strings consisting only of prefix + if base < 2 || 16 < base || (base != 8 && i >= n) { + return z, 0, 0 + } + + // convert string + z = z.make(0) + for ; i < n; i++ { + d := hexValue(s[i]) + if 0 <= d && d < base { + z = z.mulAddWW(z, Word(base), Word(d)) + } else { + break + } + } + + return z.norm(), base, i +} + + +// string converts x to a string for a given base, with 2 <= base <= 16. +// TODO(gri) in the style of the other routines, perhaps this should take +// a []byte buffer and return it +func (x nat) string(base int) string { + if base < 2 || 16 < base { + panic("illegal base") + } + + if len(x) == 0 { + return "0" + } + + // allocate buffer for conversion + i := x.bitLen()/log2(Word(base)) + 1 // +1: round up + s := make([]byte, i) + + // don't destroy x + q := nat(nil).set(x) + + // convert + for len(q) > 0 { + i-- + var r Word + q, r = q.divW(q, Word(base)) + s[i] = "0123456789abcdef"[r] + } + + return string(s[i:]) +} + + +const deBruijn32 = 0x077CB531 + +var deBruijn32Lookup = []byte{ + 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8, + 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9, +} + +const deBruijn64 = 0x03f79d71b4ca8b09 + +var deBruijn64Lookup = []byte{ + 0, 1, 56, 2, 57, 49, 28, 3, 61, 58, 42, 50, 38, 29, 17, 4, + 62, 47, 59, 36, 45, 43, 51, 22, 53, 39, 33, 30, 24, 18, 12, 5, + 63, 55, 48, 27, 60, 41, 37, 16, 46, 35, 44, 21, 52, 32, 23, 11, + 54, 26, 40, 15, 34, 20, 31, 10, 25, 14, 19, 9, 13, 8, 7, 6, +} + +// trailingZeroBits returns the number of consecutive zero bits on the right +// side of the given Word. +// See Knuth, volume 4, section 7.3.1 +func trailingZeroBits(x Word) int { + // x & -x leaves only the right-most bit set in the word. Let k be the + // index of that bit. Since only a single bit is set, the value is two + // to the power of k. Multipling by a power of two is equivalent to + // left shifting, in this case by k bits. The de Bruijn constant is + // such that all six bit, consecutive substrings are distinct. + // Therefore, if we have a left shifted version of this constant we can + // find by how many bits it was shifted by looking at which six bit + // substring ended up at the top of the word. + switch _W { + case 32: + return int(deBruijn32Lookup[((x&-x)*deBruijn32)>>27]) + case 64: + return int(deBruijn64Lookup[((x&-x)*(deBruijn64&_M))>>58]) + default: + panic("Unknown word size") + } + + return 0 +} + + +// z = x << s +func (z nat) shl(x nat, s uint) nat { + m := len(x) + if m == 0 { + return z.make(0) + } + // m > 0 + + n := m + int(s/_W) + z = z.make(n + 1) + z[n] = shlVW(z[n-m:n], x, Word(s%_W)) + z[0 : n-m].clear() + + return z.norm() +} + + +// z = x >> s +func (z nat) shr(x nat, s uint) nat { + m := len(x) + n := m - int(s/_W) + if n <= 0 { + return z.make(0) + } + // n > 0 + + z = z.make(n) + shrVW(z, x[m-n:], Word(s%_W)) + + return z.norm() +} + + +func (z nat) and(x, y nat) nat { + m := len(x) + n := len(y) + if m > n { + m = n + } + // m <= n + + z = z.make(m) + for i := 0; i < m; i++ { + z[i] = x[i] & y[i] + } + + return z.norm() +} + + +func (z nat) andNot(x, y nat) nat { + m := len(x) + n := len(y) + if n > m { + n = m + } + // m >= n + + z = z.make(m) + for i := 0; i < n; i++ { + z[i] = x[i] &^ y[i] + } + copy(z[n:m], x[n:m]) + + return z.norm() +} + + +func (z nat) or(x, y nat) nat { + m := len(x) + n := len(y) + s := x + if m < n { + n, m = m, n + s = y + } + // m >= n + + z = z.make(m) + for i := 0; i < n; i++ { + z[i] = x[i] | y[i] + } + copy(z[n:m], s[n:m]) + + return z.norm() +} + + +func (z nat) xor(x, y nat) nat { + m := len(x) + n := len(y) + s := x + if m < n { + n, m = m, n + s = y + } + // m >= n + + z = z.make(m) + for i := 0; i < n; i++ { + z[i] = x[i] ^ y[i] + } + copy(z[n:m], s[n:m]) + + return z.norm() +} + + +// greaterThan returns true iff (x1<<_W + x2) > (y1<<_W + y2) +func greaterThan(x1, x2, y1, y2 Word) bool { return x1 > y1 || x1 == y1 && x2 > y2 } + + +// modW returns x % d. +func (x nat) modW(d Word) (r Word) { + // TODO(agl): we don't actually need to store the q value. + var q nat + q = q.make(len(x)) + return divWVW(q, 0, x, d) +} + + +// powersOfTwoDecompose finds q and k such that q * 1<= 0; i-- { + v = y[i] + + for j := 0; j < _W; j++ { + z = z.mul(z, z) + + if v&mask != 0 { + z = z.mul(z, x) + } + + if m != nil { + q, z = q.div(z, z, m) + } + + v <<= 1 + } + } + + return z +} + + +// probablyPrime performs reps Miller-Rabin tests to check whether n is prime. +// If it returns true, n is prime with probability 1 - 1/4^reps. +// If it returns false, n is not prime. +func (n nat) probablyPrime(reps int) bool { + if len(n) == 0 { + return false + } + + if len(n) == 1 { + if n[0] < 2 { + return false + } + + if n[0]%2 == 0 { + return n[0] == 2 + } + + // We have to exclude these cases because we reject all + // multiples of these numbers below. + switch n[0] { + case 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53: + return true + } + } + + const primesProduct32 = 0xC0CFD797 // Π {p ∈ primes, 2 < p <= 29} + const primesProduct64 = 0xE221F97C30E94E1D // Π {p ∈ primes, 2 < p <= 53} + + var r Word + switch _W { + case 32: + r = n.modW(primesProduct32) + case 64: + r = n.modW(primesProduct64 & _M) + default: + panic("Unknown word size") + } + + if r%3 == 0 || r%5 == 0 || r%7 == 0 || r%11 == 0 || + r%13 == 0 || r%17 == 0 || r%19 == 0 || r%23 == 0 || r%29 == 0 { + return false + } + + if _W == 64 && (r%31 == 0 || r%37 == 0 || r%41 == 0 || + r%43 == 0 || r%47 == 0 || r%53 == 0) { + return false + } + + nm1 := nat(nil).sub(n, natOne) + // 1<> 1 + for i := 0; i <= _W; i++ { + if int(leadingZeros(x)) != i { + t.Errorf("failed at %x: got %d want %d", x, leadingZeros(x), i) + } + x >>= 1 + } +} + + +type shiftTest struct { + in nat + shift uint + out nat +} + + +var leftShiftTests = []shiftTest{ + {nil, 0, nil}, + {nil, 1, nil}, + {natOne, 0, natOne}, + {natOne, 1, natTwo}, + {nat{1 << (_W - 1)}, 1, nat{0}}, + {nat{1 << (_W - 1), 0}, 1, nat{0, 1}}, +} + + +func TestShiftLeft(t *testing.T) { + for i, test := range leftShiftTests { + var z nat + z = z.shl(test.in, test.shift) + for j, d := range test.out { + if j >= len(z) || z[j] != d { + t.Errorf("#%d: got: %v want: %v", i, z, test.out) + break + } + } + } +} + + +var rightShiftTests = []shiftTest{ + {nil, 0, nil}, + {nil, 1, nil}, + {natOne, 0, natOne}, + {natOne, 1, nil}, + {natTwo, 1, natOne}, + {nat{0, 1}, 1, nat{1 << (_W - 1)}}, + {nat{2, 1, 1}, 1, nat{1<<(_W-1) + 1, 1 << (_W - 1)}}, +} + + +func TestShiftRight(t *testing.T) { + for i, test := range rightShiftTests { + var z nat + z = z.shr(test.in, test.shift) + for j, d := range test.out { + if j >= len(z) || z[j] != d { + t.Errorf("#%d: got: %v want: %v", i, z, test.out) + break + } + } + } +} + + +type modWTest struct { + in string + dividend string + out string +} + + +var modWTests32 = []modWTest{ + {"23492635982634928349238759823742", "252341", "220170"}, +} + + +var modWTests64 = []modWTest{ + {"6527895462947293856291561095690465243862946", "524326975699234", "375066989628668"}, +} + + +func runModWTests(t *testing.T, tests []modWTest) { + for i, test := range tests { + in, _ := new(Int).SetString(test.in, 10) + d, _ := new(Int).SetString(test.dividend, 10) + out, _ := new(Int).SetString(test.out, 10) + + r := in.abs.modW(d.abs[0]) + if r != out.abs[0] { + t.Errorf("#%d failed: got %s want %s", i, r, out) + } + } +} + + +func TestModW(t *testing.T) { + if _W >= 32 { + runModWTests(t, modWTests32) + } + if _W >= 64 { + runModWTests(t, modWTests64) + } +} + + +func TestTrailingZeroBits(t *testing.T) { + var x Word + x-- + for i := 0; i < _W; i++ { + if trailingZeroBits(x) != i { + t.Errorf("Failed at step %d: x: %x got: %d", i, x, trailingZeroBits(x)) + } + x <<= 1 + } +} + + +var expNNTests = []struct { + x, y, m string + out string +}{ + {"0x8000000000000000", "2", "", "0x40000000000000000000000000000000"}, + {"0x8000000000000000", "2", "6719", "4944"}, + {"0x8000000000000000", "3", "6719", "5447"}, + {"0x8000000000000000", "1000", "6719", "1603"}, + {"0x8000000000000000", "1000000", "6719", "3199"}, + { + "2938462938472983472983659726349017249287491026512746239764525612965293865296239471239874193284792387498274256129746192347", + "298472983472983471903246121093472394872319615612417471234712061", + "29834729834729834729347290846729561262544958723956495615629569234729836259263598127342374289365912465901365498236492183464", + "23537740700184054162508175125554701713153216681790245129157191391322321508055833908509185839069455749219131480588829346291", + }, +} + + +func TestExpNN(t *testing.T) { + for i, test := range expNNTests { + x, _, _ := nat(nil).scan(test.x, 0) + y, _, _ := nat(nil).scan(test.y, 0) + out, _, _ := nat(nil).scan(test.out, 0) + + var m nat + + if len(test.m) > 0 { + m, _, _ = nat(nil).scan(test.m, 0) + } + + z := nat(nil).expNN(x, y, m) + if z.cmp(out) != 0 { + t.Errorf("#%d got %v want %v", i, z, out) + } + } +} diff --git a/libgo/go/big/rat.go b/libgo/go/big/rat.go new file mode 100644 index 000000000..e70673a1c --- /dev/null +++ b/libgo/go/big/rat.go @@ -0,0 +1,326 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements multi-precision rational numbers. + +package big + +import "strings" + +// A Rat represents a quotient a/b of arbitrary precision. The zero value for +// a Rat, 0/0, is not a legal Rat. +type Rat struct { + a Int + b nat +} + + +// NewRat creates a new Rat with numerator a and denominator b. +func NewRat(a, b int64) *Rat { + return new(Rat).SetFrac64(a, b) +} + + +// SetFrac sets z to a/b and returns z. +func (z *Rat) SetFrac(a, b *Int) *Rat { + z.a.Set(a) + z.a.neg = a.neg != b.neg + z.b = z.b.set(b.abs) + return z.norm() +} + + +// SetFrac64 sets z to a/b and returns z. +func (z *Rat) SetFrac64(a, b int64) *Rat { + z.a.SetInt64(a) + if b < 0 { + b = -b + z.a.neg = !z.a.neg + } + z.b = z.b.setUint64(uint64(b)) + return z.norm() +} + + +// SetInt sets z to x (by making a copy of x) and returns z. +func (z *Rat) SetInt(x *Int) *Rat { + z.a.Set(x) + z.b = z.b.setWord(1) + return z +} + + +// SetInt64 sets z to x and returns z. +func (z *Rat) SetInt64(x int64) *Rat { + z.a.SetInt64(x) + z.b = z.b.setWord(1) + return z +} + + +// Sign returns: +// +// -1 if x < 0 +// 0 if x == 0 +// +1 if x > 0 +// +func (x *Rat) Sign() int { + return x.a.Sign() +} + + +// IsInt returns true if the denominator of x is 1. +func (x *Rat) IsInt() bool { + return len(x.b) == 1 && x.b[0] == 1 +} + + +// Num returns the numerator of z; it may be <= 0. +// The result is a reference to z's numerator; it +// may change if a new value is assigned to z. +func (z *Rat) Num() *Int { + return &z.a +} + + +// Demom returns the denominator of z; it is always > 0. +// The result is a reference to z's denominator; it +// may change if a new value is assigned to z. +func (z *Rat) Denom() *Int { + return &Int{false, z.b} +} + + +func gcd(x, y nat) nat { + // Euclidean algorithm. + var a, b nat + a = a.set(x) + b = b.set(y) + for len(b) != 0 { + var q, r nat + _, r = q.div(r, a, b) + a = b + b = r + } + return a +} + + +func (z *Rat) norm() *Rat { + f := gcd(z.a.abs, z.b) + if len(z.a.abs) == 0 { + // z == 0 + z.a.neg = false // normalize sign + z.b = z.b.setWord(1) + return z + } + if f.cmp(natOne) != 0 { + z.a.abs, _ = z.a.abs.div(nil, z.a.abs, f) + z.b, _ = z.b.div(nil, z.b, f) + } + return z +} + + +func mulNat(x *Int, y nat) *Int { + var z Int + z.abs = z.abs.mul(x.abs, y) + z.neg = len(z.abs) > 0 && x.neg + return &z +} + + +// Cmp compares x and y and returns: +// +// -1 if x < y +// 0 if x == y +// +1 if x > y +// +func (x *Rat) Cmp(y *Rat) (r int) { + return mulNat(&x.a, y.b).Cmp(mulNat(&y.a, x.b)) +} + + +// Abs sets z to |x| (the absolute value of x) and returns z. +func (z *Rat) Abs(x *Rat) *Rat { + z.a.Abs(&x.a) + z.b = z.b.set(x.b) + return z +} + + +// Add sets z to the sum x+y and returns z. +func (z *Rat) Add(x, y *Rat) *Rat { + a1 := mulNat(&x.a, y.b) + a2 := mulNat(&y.a, x.b) + z.a.Add(a1, a2) + z.b = z.b.mul(x.b, y.b) + return z.norm() +} + + +// Sub sets z to the difference x-y and returns z. +func (z *Rat) Sub(x, y *Rat) *Rat { + a1 := mulNat(&x.a, y.b) + a2 := mulNat(&y.a, x.b) + z.a.Sub(a1, a2) + z.b = z.b.mul(x.b, y.b) + return z.norm() +} + + +// Mul sets z to the product x*y and returns z. +func (z *Rat) Mul(x, y *Rat) *Rat { + z.a.Mul(&x.a, &y.a) + z.b = z.b.mul(x.b, y.b) + return z.norm() +} + + +// Quo sets z to the quotient x/y and returns z. +// If y == 0, a division-by-zero run-time panic occurs. +func (z *Rat) Quo(x, y *Rat) *Rat { + if len(y.a.abs) == 0 { + panic("division by zero") + } + a := mulNat(&x.a, y.b) + b := mulNat(&y.a, x.b) + z.a.abs = a.abs + z.b = b.abs + z.a.neg = a.neg != b.neg + return z.norm() +} + + +// Neg sets z to -x (by making a copy of x if necessary) and returns z. +func (z *Rat) Neg(x *Rat) *Rat { + z.a.Neg(&x.a) + z.b = z.b.set(x.b) + return z +} + + +// Set sets z to x (by making a copy of x if necessary) and returns z. +func (z *Rat) Set(x *Rat) *Rat { + z.a.Set(&x.a) + z.b = z.b.set(x.b) + return z +} + + +// SetString sets z to the value of s and returns z and a boolean indicating +// success. s can be given as a fraction "a/b" or as a floating-point number +// optionally followed by an exponent. If the operation failed, the value of z +// is undefined. +func (z *Rat) SetString(s string) (*Rat, bool) { + if len(s) == 0 { + return z, false + } + + // check for a quotient + sep := strings.Index(s, "/") + if sep >= 0 { + if _, ok := z.a.SetString(s[0:sep], 10); !ok { + return z, false + } + s = s[sep+1:] + var n int + if z.b, _, n = z.b.scan(s, 10); n != len(s) { + return z, false + } + return z.norm(), true + } + + // check for a decimal point + sep = strings.Index(s, ".") + // check for an exponent + e := strings.IndexAny(s, "eE") + var exp Int + if e >= 0 { + if e < sep { + // The E must come after the decimal point. + return z, false + } + if _, ok := exp.SetString(s[e+1:], 10); !ok { + return z, false + } + s = s[0:e] + } + if sep >= 0 { + s = s[0:sep] + s[sep+1:] + exp.Sub(&exp, NewInt(int64(len(s)-sep))) + } + + if _, ok := z.a.SetString(s, 10); !ok { + return z, false + } + powTen := nat{}.expNN(natTen, exp.abs, nil) + if exp.neg { + z.b = powTen + z.norm() + } else { + z.a.abs = z.a.abs.mul(z.a.abs, powTen) + z.b = z.b.setWord(1) + } + + return z, true +} + + +// String returns a string representation of z in the form "a/b" (even if b == 1). +func (z *Rat) String() string { + return z.a.String() + "/" + z.b.string(10) +} + + +// RatString returns a string representation of z in the form "a/b" if b != 1, +// and in the form "a" if b == 1. +func (z *Rat) RatString() string { + if z.IsInt() { + return z.a.String() + } + return z.String() +} + + +// FloatString returns a string representation of z in decimal form with prec +// digits of precision after the decimal point and the last digit rounded. +func (z *Rat) FloatString(prec int) string { + if z.IsInt() { + return z.a.String() + } + + q, r := nat{}.div(nat{}, z.a.abs, z.b) + + p := natOne + if prec > 0 { + p = nat{}.expNN(natTen, nat{}.setUint64(uint64(prec)), nil) + } + + r = r.mul(r, p) + r, r2 := r.div(nat{}, r, z.b) + + // see if we need to round up + r2 = r2.add(r2, r2) + if z.b.cmp(r2) <= 0 { + r = r.add(r, natOne) + if r.cmp(p) >= 0 { + q = nat{}.add(q, natOne) + r = nat{}.sub(r, p) + } + } + + s := q.string(10) + if z.a.neg { + s = "-" + s + } + + if prec > 0 { + rs := r.string(10) + leadingZeros := prec - len(rs) + s += "." + strings.Repeat("0", leadingZeros) + rs + } + + return s +} diff --git a/libgo/go/big/rat_test.go b/libgo/go/big/rat_test.go new file mode 100644 index 000000000..8f42949b0 --- /dev/null +++ b/libgo/go/big/rat_test.go @@ -0,0 +1,282 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package big + +import "testing" + + +var setStringTests = []struct { + in, out string + ok bool +}{ + {"0", "0", true}, + {"-0", "0", true}, + {"1", "1", true}, + {"-1", "-1", true}, + {"1.", "1", true}, + {"1e0", "1", true}, + {"1.e1", "10", true}, + {in: "1e", ok: false}, + {in: "1.e", ok: false}, + {in: "1e+14e-5", ok: false}, + {in: "1e4.5", ok: false}, + {in: "r", ok: false}, + {in: "a/b", ok: false}, + {in: "a.b", ok: false}, + {"-0.1", "-1/10", true}, + {"-.1", "-1/10", true}, + {"2/4", "1/2", true}, + {".25", "1/4", true}, + {"-1/5", "-1/5", true}, + {"8129567.7690E14", "812956776900000000000", true}, + {"78189e+4", "781890000", true}, + {"553019.8935e+8", "55301989350000", true}, + {"98765432109876543210987654321e-10", "98765432109876543210987654321/10000000000", true}, + {"9877861857500000E-7", "3951144743/4", true}, + {"2169378.417e-3", "2169378417/1000000", true}, + {"884243222337379604041632732738665534", "884243222337379604041632732738665534", true}, + {"53/70893980658822810696", "53/70893980658822810696", true}, + {"106/141787961317645621392", "53/70893980658822810696", true}, + {"204211327800791583.81095", "4084226556015831676219/20000", true}, +} + +func TestRatSetString(t *testing.T) { + for i, test := range setStringTests { + x, ok := new(Rat).SetString(test.in) + + if ok != test.ok || ok && x.RatString() != test.out { + t.Errorf("#%d got %s want %s", i, x.RatString(), test.out) + } + } +} + + +var floatStringTests = []struct { + in string + prec int + out string +}{ + {"0", 0, "0"}, + {"0", 4, "0"}, + {"1", 0, "1"}, + {"1", 2, "1"}, + {"-1", 0, "-1"}, + {".25", 2, "0.25"}, + {".25", 1, "0.3"}, + {"-1/3", 3, "-0.333"}, + {"-2/3", 4, "-0.6667"}, + {"0.96", 1, "1.0"}, + {"0.999", 2, "1.00"}, + {"0.9", 0, "1"}, + {".25", -1, "0"}, + {".55", -1, "1"}, +} + +func TestFloatString(t *testing.T) { + for i, test := range floatStringTests { + x, _ := new(Rat).SetString(test.in) + + if x.FloatString(test.prec) != test.out { + t.Errorf("#%d got %s want %s", i, x.FloatString(test.prec), test.out) + } + } +} + + +func TestRatSign(t *testing.T) { + zero := NewRat(0, 1) + for _, a := range setStringTests { + var x Rat + x.SetString(a.in) + s := x.Sign() + e := x.Cmp(zero) + if s != e { + t.Errorf("got %d; want %d for z = %v", s, e, &x) + } + } +} + + +var ratCmpTests = []struct { + rat1, rat2 string + out int +}{ + {"0", "0/1", 0}, + {"1/1", "1", 0}, + {"-1", "-2/2", 0}, + {"1", "0", 1}, + {"0/1", "1/1", -1}, + {"-5/1434770811533343057144", "-5/1434770811533343057145", -1}, + {"49832350382626108453/8964749413", "49832350382626108454/8964749413", -1}, + {"-37414950961700930/7204075375675961", "37414950961700930/7204075375675961", -1}, + {"37414950961700930/7204075375675961", "74829901923401860/14408150751351922", 0}, +} + +func TestRatCmp(t *testing.T) { + for i, test := range ratCmpTests { + x, _ := new(Rat).SetString(test.rat1) + y, _ := new(Rat).SetString(test.rat2) + + out := x.Cmp(y) + if out != test.out { + t.Errorf("#%d got out = %v; want %v", i, out, test.out) + } + } +} + + +func TestIsInt(t *testing.T) { + one := NewInt(1) + for _, a := range setStringTests { + var x Rat + x.SetString(a.in) + i := x.IsInt() + e := x.Denom().Cmp(one) == 0 + if i != e { + t.Errorf("got %v; want %v for z = %v", i, e, &x) + } + } +} + + +func TestRatAbs(t *testing.T) { + zero := NewRat(0, 1) + for _, a := range setStringTests { + var z Rat + z.SetString(a.in) + var e Rat + e.Set(&z) + if e.Cmp(zero) < 0 { + e.Sub(zero, &e) + } + z.Abs(&z) + if z.Cmp(&e) != 0 { + t.Errorf("got z = %v; want %v", &z, &e) + } + } +} + + +type ratBinFun func(z, x, y *Rat) *Rat +type ratBinArg struct { + x, y, z string +} + +func testRatBin(t *testing.T, i int, name string, f ratBinFun, a ratBinArg) { + x, _ := NewRat(0, 1).SetString(a.x) + y, _ := NewRat(0, 1).SetString(a.y) + z, _ := NewRat(0, 1).SetString(a.z) + out := f(NewRat(0, 1), x, y) + + if out.Cmp(z) != 0 { + t.Errorf("%s #%d got %s want %s", name, i, out, z) + } +} + + +var ratBinTests = []struct { + x, y string + sum, prod string +}{ + {"0", "0", "0", "0"}, + {"0", "1", "1", "0"}, + {"-1", "0", "-1", "0"}, + {"-1", "1", "0", "-1"}, + {"1", "1", "2", "1"}, + {"1/2", "1/2", "1", "1/4"}, + {"1/4", "1/3", "7/12", "1/12"}, + {"2/5", "-14/3", "-64/15", "-28/15"}, + {"4707/49292519774798173060", "-3367/70976135186689855734", "84058377121001851123459/1749296273614329067191168098769082663020", "-1760941/388732505247628681598037355282018369560"}, + {"-61204110018146728334/3", "-31052192278051565633/2", "-215564796870448153567/6", "950260896245257153059642991192710872711/3"}, + {"-854857841473707320655/4237645934602118692642972629634714039", "-18/31750379913563777419", "-27/133467566250814981", "15387441146526731771790/134546868362786310073779084329032722548987800600710485341"}, + {"618575745270541348005638912139/19198433543745179392300736", "-19948846211000086/637313996471", "27674141753240653/30123979153216", "-6169936206128396568797607742807090270137721977/6117715203873571641674006593837351328"}, + {"-3/26206484091896184128", "5/2848423294177090248", "15310893822118706237/9330894968229805033368778458685147968", "-5/24882386581946146755650075889827061248"}, + {"26946729/330400702820", "41563965/225583428284", "1238218672302860271/4658307703098666660055", "224002580204097/14906584649915733312176"}, + {"-8259900599013409474/7", "-84829337473700364773/56707961321161574960", "-468402123685491748914621885145127724451/396955729248131024720", "350340947706464153265156004876107029701/198477864624065512360"}, + {"575775209696864/1320203974639986246357", "29/712593081308", "410331716733912717985762465/940768218243776489278275419794956", "808/45524274987585732633"}, + {"1786597389946320496771/2066653520653241", "6269770/1992362624741777", "3559549865190272133656109052308126637/4117523232840525481453983149257", "8967230/3296219033"}, + {"-36459180403360509753/32150500941194292113930", "9381566963714/9633539", "301622077145533298008420642898530153/309723104686531919656937098270", "-3784609207827/3426986245"}, +} + +func TestRatBin(t *testing.T) { + for i, test := range ratBinTests { + arg := ratBinArg{test.x, test.y, test.sum} + testRatBin(t, i, "Add", (*Rat).Add, arg) + + arg = ratBinArg{test.y, test.x, test.sum} + testRatBin(t, i, "Add symmetric", (*Rat).Add, arg) + + arg = ratBinArg{test.sum, test.x, test.y} + testRatBin(t, i, "Sub", (*Rat).Sub, arg) + + arg = ratBinArg{test.sum, test.y, test.x} + testRatBin(t, i, "Sub symmetric", (*Rat).Sub, arg) + + arg = ratBinArg{test.x, test.y, test.prod} + testRatBin(t, i, "Mul", (*Rat).Mul, arg) + + arg = ratBinArg{test.y, test.x, test.prod} + testRatBin(t, i, "Mul symmetric", (*Rat).Mul, arg) + + if test.x != "0" { + arg = ratBinArg{test.prod, test.x, test.y} + testRatBin(t, i, "Quo", (*Rat).Quo, arg) + } + + if test.y != "0" { + arg = ratBinArg{test.prod, test.y, test.x} + testRatBin(t, i, "Quo symmetric", (*Rat).Quo, arg) + } + } +} + + +func TestIssue820(t *testing.T) { + x := NewRat(3, 1) + y := NewRat(2, 1) + z := y.Quo(x, y) + q := NewRat(3, 2) + if z.Cmp(q) != 0 { + t.Errorf("got %s want %s", z, q) + } + + y = NewRat(3, 1) + x = NewRat(2, 1) + z = y.Quo(x, y) + q = NewRat(2, 3) + if z.Cmp(q) != 0 { + t.Errorf("got %s want %s", z, q) + } + + x = NewRat(3, 1) + z = x.Quo(x, x) + q = NewRat(3, 3) + if z.Cmp(q) != 0 { + t.Errorf("got %s want %s", z, q) + } +} + + +var setFrac64Tests = []struct { + a, b int64 + out string +}{ + {0, 1, "0"}, + {0, -1, "0"}, + {1, 1, "1"}, + {-1, 1, "-1"}, + {1, -1, "-1"}, + {-1, -1, "1"}, + {-9223372036854775808, -9223372036854775808, "1"}, +} + +func TestRatSetFrac64Rat(t *testing.T) { + for i, test := range setFrac64Tests { + x := new(Rat).SetFrac64(test.a, test.b) + if x.RatString() != test.out { + t.Errorf("#%d got %s want %s", i, x.RatString(), test.out) + } + } +} diff --git a/libgo/go/bufio/bufio.go b/libgo/go/bufio/bufio.go new file mode 100644 index 000000000..c13456a63 --- /dev/null +++ b/libgo/go/bufio/bufio.go @@ -0,0 +1,526 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This package implements buffered I/O. It wraps an io.Reader or io.Writer +// object, creating another object (Reader or Writer) that also implements +// the interface but provides buffering and some help for textual I/O. +package bufio + +import ( + "bytes" + "io" + "os" + "strconv" + "utf8" +) + + +const ( + defaultBufSize = 4096 +) + +// Errors introduced by this package. +type Error struct { + os.ErrorString +} + +var ( + ErrInvalidUnreadByte os.Error = &Error{"bufio: invalid use of UnreadByte"} + ErrInvalidUnreadRune os.Error = &Error{"bufio: invalid use of UnreadRune"} + ErrBufferFull os.Error = &Error{"bufio: buffer full"} + ErrNegativeCount os.Error = &Error{"bufio: negative count"} + errInternal os.Error = &Error{"bufio: internal error"} +) + +// BufSizeError is the error representing an invalid buffer size. +type BufSizeError int + +func (b BufSizeError) String() string { + return "bufio: bad buffer size " + strconv.Itoa(int(b)) +} + + +// Buffered input. + +// Reader implements buffering for an io.Reader object. +type Reader struct { + buf []byte + rd io.Reader + r, w int + err os.Error + lastByte int + lastRuneSize int +} + +// NewReaderSize creates a new Reader whose buffer has the specified size, +// which must be greater than zero. If the argument io.Reader is already a +// Reader with large enough size, it returns the underlying Reader. +// It returns the Reader and any error. +func NewReaderSize(rd io.Reader, size int) (*Reader, os.Error) { + if size <= 0 { + return nil, BufSizeError(size) + } + // Is it already a Reader? + b, ok := rd.(*Reader) + if ok && len(b.buf) >= size { + return b, nil + } + b = new(Reader) + b.buf = make([]byte, size) + b.rd = rd + b.lastByte = -1 + b.lastRuneSize = -1 + return b, nil +} + +// NewReader returns a new Reader whose buffer has the default size. +func NewReader(rd io.Reader) *Reader { + b, err := NewReaderSize(rd, defaultBufSize) + if err != nil { + // cannot happen - defaultBufSize is a valid size + panic(err) + } + return b +} + +// fill reads a new chunk into the buffer. +func (b *Reader) fill() { + // Slide existing data to beginning. + if b.r > 0 { + copy(b.buf, b.buf[b.r:b.w]) + b.w -= b.r + b.r = 0 + } + + // Read new data. + n, e := b.rd.Read(b.buf[b.w:]) + b.w += n + if e != nil { + b.err = e + } +} + +// Peek returns the next n bytes without advancing the reader. The bytes stop +// being valid at the next read call. If Peek returns fewer than n bytes, it +// also returns an error explaining why the read is short. The error is +// ErrBufferFull if n is larger than b's buffer size. +func (b *Reader) Peek(n int) ([]byte, os.Error) { + if n < 0 { + return nil, ErrNegativeCount + } + if n > len(b.buf) { + return nil, ErrBufferFull + } + for b.w-b.r < n && b.err == nil { + b.fill() + } + m := b.w - b.r + if m > n { + m = n + } + err := b.err + if m < n && err == nil { + err = ErrBufferFull + } + return b.buf[b.r : b.r+m], err +} + +// Read reads data into p. +// It returns the number of bytes read into p. +// It calls Read at most once on the underlying Reader, +// hence n may be less than len(p). +// At EOF, the count will be zero and err will be os.EOF. +func (b *Reader) Read(p []byte) (n int, err os.Error) { + n = len(p) + if n == 0 { + return 0, b.err + } + if b.w == b.r { + if b.err != nil { + return 0, b.err + } + if len(p) >= len(b.buf) { + // Large read, empty buffer. + // Read directly into p to avoid copy. + n, b.err = b.rd.Read(p) + if n > 0 { + b.lastByte = int(p[n-1]) + b.lastRuneSize = -1 + } + return n, b.err + } + b.fill() + if b.w == b.r { + return 0, b.err + } + } + + if n > b.w-b.r { + n = b.w - b.r + } + copy(p[0:n], b.buf[b.r:]) + b.r += n + b.lastByte = int(b.buf[b.r-1]) + b.lastRuneSize = -1 + return n, nil +} + +// ReadByte reads and returns a single byte. +// If no byte is available, returns an error. +func (b *Reader) ReadByte() (c byte, err os.Error) { + b.lastRuneSize = -1 + for b.w == b.r { + if b.err != nil { + return 0, b.err + } + b.fill() + } + c = b.buf[b.r] + b.r++ + b.lastByte = int(c) + return c, nil +} + +// UnreadByte unreads the last byte. Only the most recently read byte can be unread. +func (b *Reader) UnreadByte() os.Error { + b.lastRuneSize = -1 + if b.r == b.w && b.lastByte >= 0 { + b.w = 1 + b.r = 0 + b.buf[0] = byte(b.lastByte) + b.lastByte = -1 + return nil + } + if b.r <= 0 { + return ErrInvalidUnreadByte + } + b.r-- + b.lastByte = -1 + return nil +} + +// ReadRune reads a single UTF-8 encoded Unicode character and returns the +// rune and its size in bytes. +func (b *Reader) ReadRune() (rune int, size int, err os.Error) { + for b.r+utf8.UTFMax > b.w && !utf8.FullRune(b.buf[b.r:b.w]) && b.err == nil { + b.fill() + } + b.lastRuneSize = -1 + if b.r == b.w { + return 0, 0, b.err + } + rune, size = int(b.buf[b.r]), 1 + if rune >= 0x80 { + rune, size = utf8.DecodeRune(b.buf[b.r:b.w]) + } + b.r += size + b.lastByte = int(b.buf[b.r-1]) + b.lastRuneSize = size + return rune, size, nil +} + +// UnreadRune unreads the last rune. If the most recent read operation on +// the buffer was not a ReadRune, UnreadRune returns an error. (In this +// regard it is stricter than UnreadByte, which will unread the last byte +// from any read operation.) +func (b *Reader) UnreadRune() os.Error { + if b.lastRuneSize < 0 || b.r == 0 { + return ErrInvalidUnreadRune + } + b.r -= b.lastRuneSize + b.lastByte = -1 + b.lastRuneSize = -1 + return nil +} + +// Buffered returns the number of bytes that can be read from the current buffer. +func (b *Reader) Buffered() int { return b.w - b.r } + +// ReadSlice reads until the first occurrence of delim in the input, +// returning a slice pointing at the bytes in the buffer. +// The bytes stop being valid at the next read call. +// If ReadSlice encounters an error before finding a delimiter, +// it returns all the data in the buffer and the error itself (often os.EOF). +// ReadSlice fails with error ErrBufferFull if the buffer fills without a delim. +// Because the data returned from ReadSlice will be overwritten +// by the next I/O operation, most clients should use +// ReadBytes or ReadString instead. +// ReadSlice returns err != nil if and only if line does not end in delim. +func (b *Reader) ReadSlice(delim byte) (line []byte, err os.Error) { + // Look in buffer. + if i := bytes.IndexByte(b.buf[b.r:b.w], delim); i >= 0 { + line1 := b.buf[b.r : b.r+i+1] + b.r += i + 1 + return line1, nil + } + + // Read more into buffer, until buffer fills or we find delim. + for { + if b.err != nil { + line := b.buf[b.r:b.w] + b.r = b.w + return line, b.err + } + + n := b.Buffered() + b.fill() + + // Search new part of buffer + if i := bytes.IndexByte(b.buf[n:b.w], delim); i >= 0 { + line := b.buf[0 : n+i+1] + b.r = n + i + 1 + return line, nil + } + + // Buffer is full? + if b.Buffered() >= len(b.buf) { + b.r = b.w + return b.buf, ErrBufferFull + } + } + panic("not reached") +} + +// ReadBytes reads until the first occurrence of delim in the input, +// returning a slice containing the data up to and including the delimiter. +// If ReadBytes encounters an error before finding a delimiter, +// it returns the data read before the error and the error itself (often os.EOF). +// ReadBytes returns err != nil if and only if line does not end in delim. +func (b *Reader) ReadBytes(delim byte) (line []byte, err os.Error) { + // Use ReadSlice to look for array, + // accumulating full buffers. + var frag []byte + var full [][]byte + err = nil + + for { + var e os.Error + frag, e = b.ReadSlice(delim) + if e == nil { // got final fragment + break + } + if e != ErrBufferFull { // unexpected error + err = e + break + } + + // Make a copy of the buffer. + buf := make([]byte, len(frag)) + copy(buf, frag) + full = append(full, buf) + } + + // Allocate new buffer to hold the full pieces and the fragment. + n := 0 + for i := range full { + n += len(full[i]) + } + n += len(frag) + + // Copy full pieces and fragment in. + buf := make([]byte, n) + n = 0 + for i := range full { + n += copy(buf[n:], full[i]) + } + copy(buf[n:], frag) + return buf, err +} + +// ReadString reads until the first occurrence of delim in the input, +// returning a string containing the data up to and including the delimiter. +// If ReadString encounters an error before finding a delimiter, +// it returns the data read before the error and the error itself (often os.EOF). +// ReadString returns err != nil if and only if line does not end in delim. +func (b *Reader) ReadString(delim byte) (line string, err os.Error) { + bytes, e := b.ReadBytes(delim) + return string(bytes), e +} + + +// buffered output + +// Writer implements buffering for an io.Writer object. +type Writer struct { + err os.Error + buf []byte + n int + wr io.Writer +} + +// NewWriterSize creates a new Writer whose buffer has the specified size, +// which must be greater than zero. If the argument io.Writer is already a +// Writer with large enough size, it returns the underlying Writer. +// It returns the Writer and any error. +func NewWriterSize(wr io.Writer, size int) (*Writer, os.Error) { + if size <= 0 { + return nil, BufSizeError(size) + } + // Is it already a Writer? + b, ok := wr.(*Writer) + if ok && len(b.buf) >= size { + return b, nil + } + b = new(Writer) + b.buf = make([]byte, size) + b.wr = wr + return b, nil +} + +// NewWriter returns a new Writer whose buffer has the default size. +func NewWriter(wr io.Writer) *Writer { + b, err := NewWriterSize(wr, defaultBufSize) + if err != nil { + // cannot happen - defaultBufSize is valid size + panic(err) + } + return b +} + +// Flush writes any buffered data to the underlying io.Writer. +func (b *Writer) Flush() os.Error { + if b.err != nil { + return b.err + } + n, e := b.wr.Write(b.buf[0:b.n]) + if n < b.n && e == nil { + e = io.ErrShortWrite + } + if e != nil { + if n > 0 && n < b.n { + copy(b.buf[0:b.n-n], b.buf[n:b.n]) + } + b.n -= n + b.err = e + return e + } + b.n = 0 + return nil +} + +// Available returns how many bytes are unused in the buffer. +func (b *Writer) Available() int { return len(b.buf) - b.n } + +// Buffered returns the number of bytes that have been written into the current buffer. +func (b *Writer) Buffered() int { return b.n } + +// Write writes the contents of p into the buffer. +// It returns the number of bytes written. +// If nn < len(p), it also returns an error explaining +// why the write is short. +func (b *Writer) Write(p []byte) (nn int, err os.Error) { + if b.err != nil { + return 0, b.err + } + nn = 0 + for len(p) > 0 { + n := b.Available() + if n <= 0 { + if b.Flush(); b.err != nil { + break + } + n = b.Available() + } + if b.Buffered() == 0 && len(p) >= len(b.buf) { + // Large write, empty buffer. + // Write directly from p to avoid copy. + n, b.err = b.wr.Write(p) + nn += n + p = p[n:] + if b.err != nil { + break + } + continue + } + if n > len(p) { + n = len(p) + } + copy(b.buf[b.n:b.n+n], p[0:n]) + b.n += n + nn += n + p = p[n:] + } + return nn, b.err +} + +// WriteByte writes a single byte. +func (b *Writer) WriteByte(c byte) os.Error { + if b.err != nil { + return b.err + } + if b.Available() <= 0 && b.Flush() != nil { + return b.err + } + b.buf[b.n] = c + b.n++ + return nil +} + +// WriteRune writes a single Unicode code point, returning +// the number of bytes written and any error. +func (b *Writer) WriteRune(rune int) (size int, err os.Error) { + if rune < utf8.RuneSelf { + err = b.WriteByte(byte(rune)) + if err != nil { + return 0, err + } + return 1, nil + } + if b.err != nil { + return 0, b.err + } + n := b.Available() + if n < utf8.UTFMax { + if b.Flush(); b.err != nil { + return 0, b.err + } + n = b.Available() + if n < utf8.UTFMax { + // Can only happen if buffer is silly small. + return b.WriteString(string(rune)) + } + } + size = utf8.EncodeRune(b.buf[b.n:], rune) + b.n += size + return size, nil +} + +// WriteString writes a string. +// It returns the number of bytes written. +// If the count is less than len(s), it also returns an error explaining +// why the write is short. +func (b *Writer) WriteString(s string) (int, os.Error) { + if b.err != nil { + return 0, b.err + } + // Common case, worth making fast. + if b.Available() >= len(s) || len(b.buf) >= len(s) && b.Flush() == nil { + for i := 0; i < len(s); i++ { // loop over bytes, not runes. + b.buf[b.n] = s[i] + b.n++ + } + return len(s), nil + } + for i := 0; i < len(s); i++ { // loop over bytes, not runes. + b.WriteByte(s[i]) + if b.err != nil { + return i, b.err + } + } + return len(s), nil +} + +// buffered input and output + +// ReadWriter stores pointers to a Reader and a Writer. +// It implements io.ReadWriter. +type ReadWriter struct { + *Reader + *Writer +} + +// NewReadWriter allocates a new ReadWriter that dispatches to r and w. +func NewReadWriter(r *Reader, w *Writer) *ReadWriter { + return &ReadWriter{r, w} +} diff --git a/libgo/go/bufio/bufio_test.go b/libgo/go/bufio/bufio_test.go new file mode 100644 index 000000000..059ca6dd2 --- /dev/null +++ b/libgo/go/bufio/bufio_test.go @@ -0,0 +1,572 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bufio + +import ( + "bytes" + "fmt" + "io" + "os" + "strings" + "testing" + "testing/iotest" + "utf8" +) + +// Reads from a reader and rot13s the result. +type rot13Reader struct { + r io.Reader +} + +func newRot13Reader(r io.Reader) *rot13Reader { + r13 := new(rot13Reader) + r13.r = r + return r13 +} + +func (r13 *rot13Reader) Read(p []byte) (int, os.Error) { + n, e := r13.r.Read(p) + if e != nil { + return n, e + } + for i := 0; i < n; i++ { + c := p[i] | 0x20 // lowercase byte + if 'a' <= c && c <= 'm' { + p[i] += 13 + } else if 'n' <= c && c <= 'z' { + p[i] -= 13 + } + } + return n, nil +} + +// Call ReadByte to accumulate the text of a file +func readBytes(buf *Reader) string { + var b [1000]byte + nb := 0 + for { + c, e := buf.ReadByte() + if e == os.EOF { + break + } + if e != nil { + panic("Data: " + e.String()) + } + b[nb] = c + nb++ + } + return string(b[0:nb]) +} + +func TestReaderSimple(t *testing.T) { + data := "hello world" + b := NewReader(bytes.NewBufferString(data)) + if s := readBytes(b); s != "hello world" { + t.Errorf("simple hello world test failed: got %q", s) + } + + b = NewReader(newRot13Reader(bytes.NewBufferString(data))) + if s := readBytes(b); s != "uryyb jbeyq" { + t.Errorf("rot13 hello world test failed: got %q", s) + } +} + + +type readMaker struct { + name string + fn func(io.Reader) io.Reader +} + +var readMakers = []readMaker{ + {"full", func(r io.Reader) io.Reader { return r }}, + {"byte", iotest.OneByteReader}, + {"half", iotest.HalfReader}, + {"data+err", iotest.DataErrReader}, +} + +// Call ReadString (which ends up calling everything else) +// to accumulate the text of a file. +func readLines(b *Reader) string { + s := "" + for { + s1, e := b.ReadString('\n') + if e == os.EOF { + break + } + if e != nil { + panic("GetLines: " + e.String()) + } + s += s1 + } + return s +} + +// Call Read to accumulate the text of a file +func reads(buf *Reader, m int) string { + var b [1000]byte + nb := 0 + for { + n, e := buf.Read(b[nb : nb+m]) + nb += n + if e == os.EOF { + break + } + } + return string(b[0:nb]) +} + +type bufReader struct { + name string + fn func(*Reader) string +} + +var bufreaders = []bufReader{ + {"1", func(b *Reader) string { return reads(b, 1) }}, + {"2", func(b *Reader) string { return reads(b, 2) }}, + {"3", func(b *Reader) string { return reads(b, 3) }}, + {"4", func(b *Reader) string { return reads(b, 4) }}, + {"5", func(b *Reader) string { return reads(b, 5) }}, + {"7", func(b *Reader) string { return reads(b, 7) }}, + {"bytes", readBytes}, + {"lines", readLines}, +} + +var bufsizes = []int{ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, + 23, 32, 46, 64, 93, 128, 1024, 4096, +} + +func TestReader(t *testing.T) { + var texts [31]string + str := "" + all := "" + for i := 0; i < len(texts)-1; i++ { + texts[i] = str + "\n" + all += texts[i] + str += string(i%26 + 'a') + } + texts[len(texts)-1] = all + + for h := 0; h < len(texts); h++ { + text := texts[h] + for i := 0; i < len(readMakers); i++ { + for j := 0; j < len(bufreaders); j++ { + for k := 0; k < len(bufsizes); k++ { + readmaker := readMakers[i] + bufreader := bufreaders[j] + bufsize := bufsizes[k] + read := readmaker.fn(bytes.NewBufferString(text)) + buf, _ := NewReaderSize(read, bufsize) + s := bufreader.fn(buf) + if s != text { + t.Errorf("reader=%s fn=%s bufsize=%d want=%q got=%q", + readmaker.name, bufreader.name, bufsize, text, s) + } + } + } + } + } +} + +// A StringReader delivers its data one string segment at a time via Read. +type StringReader struct { + data []string + step int +} + +func (r *StringReader) Read(p []byte) (n int, err os.Error) { + if r.step < len(r.data) { + s := r.data[r.step] + n = copy(p, s) + r.step++ + } else { + err = os.EOF + } + return +} + +func readRuneSegments(t *testing.T, segments []string) { + got := "" + want := strings.Join(segments, "") + r := NewReader(&StringReader{data: segments}) + for { + rune, _, err := r.ReadRune() + if err != nil { + if err != os.EOF { + return + } + break + } + got += string(rune) + } + if got != want { + t.Errorf("segments=%v got=%s want=%s", segments, got, want) + } +} + +var segmentList = [][]string{ + {}, + {""}, + {"日", "本語"}, + {"\u65e5", "\u672c", "\u8a9e"}, + {"\U000065e5", "\U0000672c", "\U00008a9e"}, + {"\xe6", "\x97\xa5\xe6", "\x9c\xac\xe8\xaa\x9e"}, + {"Hello", ", ", "World", "!"}, + {"Hello", ", ", "", "World", "!"}, +} + +func TestReadRune(t *testing.T) { + for _, s := range segmentList { + readRuneSegments(t, s) + } +} + +func TestUnreadRune(t *testing.T) { + got := "" + segments := []string{"Hello, world:", "日本語"} + data := strings.Join(segments, "") + r := NewReader(&StringReader{data: segments}) + // Normal execution. + for { + rune, _, err := r.ReadRune() + if err != nil { + if err != os.EOF { + t.Error("unexpected EOF") + } + break + } + got += string(rune) + // Put it back and read it again + if err = r.UnreadRune(); err != nil { + t.Error("unexpected error on UnreadRune:", err) + } + rune1, _, err := r.ReadRune() + if err != nil { + t.Error("unexpected error reading after unreading:", err) + } + if rune != rune1 { + t.Errorf("incorrect rune after unread: got %c wanted %c", rune1, rune) + } + } + if got != data { + t.Errorf("want=%q got=%q", data, got) + } +} + +// Test that UnreadRune fails if the preceding operation was not a ReadRune. +func TestUnreadRuneError(t *testing.T) { + buf := make([]byte, 3) // All runes in this test are 3 bytes long + r := NewReader(&StringReader{data: []string{"日本語日本語日本語"}}) + if r.UnreadRune() == nil { + t.Error("expected error on UnreadRune from fresh buffer") + } + _, _, err := r.ReadRune() + if err != nil { + t.Error("unexpected error on ReadRune (1):", err) + } + if err = r.UnreadRune(); err != nil { + t.Error("unexpected error on UnreadRune (1):", err) + } + if r.UnreadRune() == nil { + t.Error("expected error after UnreadRune (1)") + } + // Test error after Read. + _, _, err = r.ReadRune() // reset state + if err != nil { + t.Error("unexpected error on ReadRune (2):", err) + } + _, err = r.Read(buf) + if err != nil { + t.Error("unexpected error on Read (2):", err) + } + if r.UnreadRune() == nil { + t.Error("expected error after Read (2)") + } + // Test error after ReadByte. + _, _, err = r.ReadRune() // reset state + if err != nil { + t.Error("unexpected error on ReadRune (2):", err) + } + for _ = range buf { + _, err = r.ReadByte() + if err != nil { + t.Error("unexpected error on ReadByte (2):", err) + } + } + if r.UnreadRune() == nil { + t.Error("expected error after ReadByte") + } + // Test error after UnreadByte. + _, _, err = r.ReadRune() // reset state + if err != nil { + t.Error("unexpected error on ReadRune (3):", err) + } + _, err = r.ReadByte() + if err != nil { + t.Error("unexpected error on ReadByte (3):", err) + } + err = r.UnreadByte() + if err != nil { + t.Error("unexpected error on UnreadByte (3):", err) + } + if r.UnreadRune() == nil { + t.Error("expected error after UnreadByte (3)") + } +} + +func TestUnreadRuneAtEOF(t *testing.T) { + // UnreadRune/ReadRune should error at EOF (was a bug; used to panic) + r := NewReader(strings.NewReader("x")) + r.ReadRune() + r.ReadRune() + r.UnreadRune() + _, _, err := r.ReadRune() + if err == nil { + t.Error("expected error at EOF") + } else if err != os.EOF { + t.Error("expected EOF; got", err) + } +} + +func TestReadWriteRune(t *testing.T) { + const NRune = 1000 + byteBuf := new(bytes.Buffer) + w := NewWriter(byteBuf) + // Write the runes out using WriteRune + buf := make([]byte, utf8.UTFMax) + for rune := 0; rune < NRune; rune++ { + size := utf8.EncodeRune(buf, rune) + nbytes, err := w.WriteRune(rune) + if err != nil { + t.Fatalf("WriteRune(0x%x) error: %s", rune, err) + } + if nbytes != size { + t.Fatalf("WriteRune(0x%x) expected %d, got %d", rune, size, nbytes) + } + } + w.Flush() + + r := NewReader(byteBuf) + // Read them back with ReadRune + for rune := 0; rune < NRune; rune++ { + size := utf8.EncodeRune(buf, rune) + nr, nbytes, err := r.ReadRune() + if nr != rune || nbytes != size || err != nil { + t.Fatalf("ReadRune(0x%x) got 0x%x,%d not 0x%x,%d (err=%s)", r, nr, nbytes, r, size, err) + } + } +} + +func TestWriter(t *testing.T) { + var data [8192]byte + + for i := 0; i < len(data); i++ { + data[i] = byte(' ' + i%('~'-' ')) + } + w := new(bytes.Buffer) + for i := 0; i < len(bufsizes); i++ { + for j := 0; j < len(bufsizes); j++ { + nwrite := bufsizes[i] + bs := bufsizes[j] + + // Write nwrite bytes using buffer size bs. + // Check that the right amount makes it out + // and that the data is correct. + + w.Reset() + buf, e := NewWriterSize(w, bs) + context := fmt.Sprintf("nwrite=%d bufsize=%d", nwrite, bs) + if e != nil { + t.Errorf("%s: NewWriterSize %d: %v", context, bs, e) + continue + } + n, e1 := buf.Write(data[0:nwrite]) + if e1 != nil || n != nwrite { + t.Errorf("%s: buf.Write %d = %d, %v", context, nwrite, n, e1) + continue + } + if e = buf.Flush(); e != nil { + t.Errorf("%s: buf.Flush = %v", context, e) + } + + written := w.Bytes() + if len(written) != nwrite { + t.Errorf("%s: %d bytes written", context, len(written)) + } + for l := 0; l < len(written); l++ { + if written[i] != data[i] { + t.Errorf("wrong bytes written") + t.Errorf("want=%q", data[0:len(written)]) + t.Errorf("have=%q", written) + } + } + } + } +} + +// Check that write errors are returned properly. + +type errorWriterTest struct { + n, m int + err os.Error + expect os.Error +} + +func (w errorWriterTest) Write(p []byte) (int, os.Error) { + return len(p) * w.n / w.m, w.err +} + +var errorWriterTests = []errorWriterTest{ + {0, 1, nil, io.ErrShortWrite}, + {1, 2, nil, io.ErrShortWrite}, + {1, 1, nil, nil}, + {0, 1, os.EPIPE, os.EPIPE}, + {1, 2, os.EPIPE, os.EPIPE}, + {1, 1, os.EPIPE, os.EPIPE}, +} + +func TestWriteErrors(t *testing.T) { + for _, w := range errorWriterTests { + buf := NewWriter(w) + _, e := buf.Write([]byte("hello world")) + if e != nil { + t.Errorf("Write hello to %v: %v", w, e) + continue + } + e = buf.Flush() + if e != w.expect { + t.Errorf("Flush %v: got %v, wanted %v", w, e, w.expect) + } + } +} + +func TestNewReaderSizeIdempotent(t *testing.T) { + const BufSize = 1000 + b, err := NewReaderSize(bytes.NewBufferString("hello world"), BufSize) + if err != nil { + t.Error("NewReaderSize create fail", err) + } + // Does it recognize itself? + b1, err2 := NewReaderSize(b, BufSize) + if err2 != nil { + t.Error("NewReaderSize #2 create fail", err2) + } + if b1 != b { + t.Error("NewReaderSize did not detect underlying Reader") + } + // Does it wrap if existing buffer is too small? + b2, err3 := NewReaderSize(b, 2*BufSize) + if err3 != nil { + t.Error("NewReaderSize #3 create fail", err3) + } + if b2 == b { + t.Error("NewReaderSize did not enlarge buffer") + } +} + +func TestNewWriterSizeIdempotent(t *testing.T) { + const BufSize = 1000 + b, err := NewWriterSize(new(bytes.Buffer), BufSize) + if err != nil { + t.Error("NewWriterSize create fail", err) + } + // Does it recognize itself? + b1, err2 := NewWriterSize(b, BufSize) + if err2 != nil { + t.Error("NewWriterSize #2 create fail", err2) + } + if b1 != b { + t.Error("NewWriterSize did not detect underlying Writer") + } + // Does it wrap if existing buffer is too small? + b2, err3 := NewWriterSize(b, 2*BufSize) + if err3 != nil { + t.Error("NewWriterSize #3 create fail", err3) + } + if b2 == b { + t.Error("NewWriterSize did not enlarge buffer") + } +} + +func TestWriteString(t *testing.T) { + const BufSize = 8 + buf := new(bytes.Buffer) + b, err := NewWriterSize(buf, BufSize) + if err != nil { + t.Error("NewWriterSize create fail", err) + } + b.WriteString("0") // easy + b.WriteString("123456") // still easy + b.WriteString("7890") // easy after flush + b.WriteString("abcdefghijklmnopqrstuvwxy") // hard + b.WriteString("z") + b.Flush() + if b.err != nil { + t.Error("WriteString", b.err) + } + s := "01234567890abcdefghijklmnopqrstuvwxyz" + if string(buf.Bytes()) != s { + t.Errorf("WriteString wants %q gets %q", s, string(buf.Bytes())) + } +} + +func TestBufferFull(t *testing.T) { + buf, _ := NewReaderSize(strings.NewReader("hello, world"), 5) + line, err := buf.ReadSlice(',') + if string(line) != "hello" || err != ErrBufferFull { + t.Errorf("first ReadSlice(,) = %q, %v", line, err) + } + line, err = buf.ReadSlice(',') + if string(line) != "," || err != nil { + t.Errorf("second ReadSlice(,) = %q, %v", line, err) + } +} + +func TestPeek(t *testing.T) { + p := make([]byte, 10) + buf, _ := NewReaderSize(strings.NewReader("abcdefghij"), 4) + if s, err := buf.Peek(1); string(s) != "a" || err != nil { + t.Fatalf("want %q got %q, err=%v", "a", string(s), err) + } + if s, err := buf.Peek(4); string(s) != "abcd" || err != nil { + t.Fatalf("want %q got %q, err=%v", "abcd", string(s), err) + } + if _, err := buf.Peek(5); err != ErrBufferFull { + t.Fatalf("want ErrBufFull got %v", err) + } + if _, err := buf.Read(p[0:3]); string(p[0:3]) != "abc" || err != nil { + t.Fatalf("want %q got %q, err=%v", "abc", string(p[0:3]), err) + } + if s, err := buf.Peek(1); string(s) != "d" || err != nil { + t.Fatalf("want %q got %q, err=%v", "d", string(s), err) + } + if s, err := buf.Peek(2); string(s) != "de" || err != nil { + t.Fatalf("want %q got %q, err=%v", "de", string(s), err) + } + if _, err := buf.Read(p[0:3]); string(p[0:3]) != "def" || err != nil { + t.Fatalf("want %q got %q, err=%v", "def", string(p[0:3]), err) + } + if s, err := buf.Peek(4); string(s) != "ghij" || err != nil { + t.Fatalf("want %q got %q, err=%v", "ghij", string(s), err) + } + if _, err := buf.Read(p[0:4]); string(p[0:4]) != "ghij" || err != nil { + t.Fatalf("want %q got %q, err=%v", "ghij", string(p[0:3]), err) + } + if s, err := buf.Peek(0); string(s) != "" || err != nil { + t.Fatalf("want %q got %q, err=%v", "", string(s), err) + } + if _, err := buf.Peek(1); err != os.EOF { + t.Fatalf("want EOF got %v", err) + } +} + +func TestPeekThenUnreadRune(t *testing.T) { + // This sequence used to cause a crash. + r := NewReader(strings.NewReader("x")) + r.ReadRune() + r.Peek(1) + r.UnreadRune() + r.ReadRune() // Used to panic here +} diff --git a/libgo/go/bytes/buffer.go b/libgo/go/bytes/buffer.go new file mode 100644 index 000000000..62cf82810 --- /dev/null +++ b/libgo/go/bytes/buffer.go @@ -0,0 +1,315 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bytes + +// Simple byte buffer for marshaling data. + +import ( + "io" + "os" + "utf8" +) + +// A Buffer is a variable-sized buffer of bytes with Read and Write methods. +// The zero value for Buffer is an empty buffer ready to use. +type Buffer struct { + buf []byte // contents are the bytes buf[off : len(buf)] + off int // read at &buf[off], write at &buf[len(buf)] + runeBytes [utf8.UTFMax]byte // avoid allocation of slice on each WriteByte or Rune + bootstrap [64]byte // memory to hold first slice; helps small buffers (Printf) avoid allocation. + lastRead readOp // last read operation, so that Unread* can work correctly. +} + +// The readOp constants describe the last action performed on +// the buffer, so that UnreadRune and UnreadByte can +// check for invalid usage. +type readOp int + +const ( + opInvalid readOp = iota // Non-read operation. + opReadRune // Read rune. + opRead // Any other read operation. +) + +// Bytes returns a slice of the contents of the unread portion of the buffer; +// len(b.Bytes()) == b.Len(). If the caller changes the contents of the +// returned slice, the contents of the buffer will change provided there +// are no intervening method calls on the Buffer. +func (b *Buffer) Bytes() []byte { return b.buf[b.off:] } + +// String returns the contents of the unread portion of the buffer +// as a string. If the Buffer is a nil pointer, it returns "". +func (b *Buffer) String() string { + if b == nil { + // Special case, useful in debugging. + return "" + } + return string(b.buf[b.off:]) +} + +// Len returns the number of bytes of the unread portion of the buffer; +// b.Len() == len(b.Bytes()). +func (b *Buffer) Len() int { return len(b.buf) - b.off } + +// Truncate discards all but the first n unread bytes from the buffer. +// It is an error to call b.Truncate(n) with n > b.Len(). +func (b *Buffer) Truncate(n int) { + b.lastRead = opInvalid + if n == 0 { + // Reuse buffer space. + b.off = 0 + } + b.buf = b.buf[0 : b.off+n] +} + +// Reset resets the buffer so it has no content. +// b.Reset() is the same as b.Truncate(0). +func (b *Buffer) Reset() { b.Truncate(0) } + +// Grow buffer to guarantee space for n more bytes. +// Return index where bytes should be written. +func (b *Buffer) grow(n int) int { + m := b.Len() + // If buffer is empty, reset to recover space. + if m == 0 && b.off != 0 { + b.Truncate(0) + } + if len(b.buf)+n > cap(b.buf) { + var buf []byte + if b.buf == nil && n <= len(b.bootstrap) { + buf = b.bootstrap[0:] + } else { + // not enough space anywhere + buf = make([]byte, 2*cap(b.buf)+n) + copy(buf, b.buf[b.off:]) + } + b.buf = buf + b.off = 0 + } + b.buf = b.buf[0 : b.off+m+n] + return b.off + m +} + +// Write appends the contents of p to the buffer. The return +// value n is the length of p; err is always nil. +func (b *Buffer) Write(p []byte) (n int, err os.Error) { + b.lastRead = opInvalid + m := b.grow(len(p)) + copy(b.buf[m:], p) + return len(p), nil +} + +// WriteString appends the contents of s to the buffer. The return +// value n is the length of s; err is always nil. +func (b *Buffer) WriteString(s string) (n int, err os.Error) { + b.lastRead = opInvalid + m := b.grow(len(s)) + return copy(b.buf[m:], s), nil +} + +// MinRead is the minimum slice size passed to a Read call by +// Buffer.ReadFrom. As long as the Buffer has at least MinRead bytes beyond +// what is required to hold the contents of r, ReadFrom will not grow the +// underlying buffer. +const MinRead = 512 + +// ReadFrom reads data from r until EOF and appends it to the buffer. +// The return value n is the number of bytes read. +// Any error except os.EOF encountered during the read +// is also returned. +func (b *Buffer) ReadFrom(r io.Reader) (n int64, err os.Error) { + b.lastRead = opInvalid + // If buffer is empty, reset to recover space. + if b.off >= len(b.buf) { + b.Truncate(0) + } + for { + if cap(b.buf)-len(b.buf) < MinRead { + var newBuf []byte + // can we get space without allocation? + if b.off+cap(b.buf)-len(b.buf) >= MinRead { + // reuse beginning of buffer + newBuf = b.buf[0 : len(b.buf)-b.off] + } else { + // not enough space at end; put space on end + newBuf = make([]byte, len(b.buf)-b.off, 2*(cap(b.buf)-b.off)+MinRead) + } + copy(newBuf, b.buf[b.off:]) + b.buf = newBuf + b.off = 0 + } + m, e := r.Read(b.buf[len(b.buf):cap(b.buf)]) + b.buf = b.buf[0 : len(b.buf)+m] + n += int64(m) + if e == os.EOF { + break + } + if e != nil { + return n, e + } + } + return n, nil // err is EOF, so return nil explicitly +} + +// WriteTo writes data to w until the buffer is drained or an error +// occurs. The return value n is the number of bytes written. +// Any error encountered during the write is also returned. +func (b *Buffer) WriteTo(w io.Writer) (n int64, err os.Error) { + b.lastRead = opInvalid + for b.off < len(b.buf) { + m, e := w.Write(b.buf[b.off:]) + n += int64(m) + b.off += m + if e != nil { + return n, e + } + } + // Buffer is now empty; reset. + b.Truncate(0) + return +} + +// WriteByte appends the byte c to the buffer. +// The returned error is always nil, but is included +// to match bufio.Writer's WriteByte. +func (b *Buffer) WriteByte(c byte) os.Error { + b.lastRead = opInvalid + m := b.grow(1) + b.buf[m] = c + return nil +} + +// WriteRune appends the UTF-8 encoding of Unicode +// code point r to the buffer, returning its length and +// an error, which is always nil but is included +// to match bufio.Writer's WriteRune. +func (b *Buffer) WriteRune(r int) (n int, err os.Error) { + if r < utf8.RuneSelf { + b.WriteByte(byte(r)) + return 1, nil + } + n = utf8.EncodeRune(b.runeBytes[0:], r) + b.Write(b.runeBytes[0:n]) + return n, nil +} + +// Read reads the next len(p) bytes from the buffer or until the buffer +// is drained. The return value n is the number of bytes read. If the +// buffer has no data to return, err is os.EOF even if len(p) is zero; +// otherwise it is nil. +func (b *Buffer) Read(p []byte) (n int, err os.Error) { + b.lastRead = opInvalid + if b.off >= len(b.buf) { + // Buffer is empty, reset to recover space. + b.Truncate(0) + return 0, os.EOF + } + n = copy(p, b.buf[b.off:]) + b.off += n + if n > 0 { + b.lastRead = opRead + } + return +} + +// Next returns a slice containing the next n bytes from the buffer, +// advancing the buffer as if the bytes had been returned by Read. +// If there are fewer than n bytes in the buffer, Next returns the entire buffer. +// The slice is only valid until the next call to a read or write method. +func (b *Buffer) Next(n int) []byte { + b.lastRead = opInvalid + m := b.Len() + if n > m { + n = m + } + data := b.buf[b.off : b.off+n] + b.off += n + if n > 0 { + b.lastRead = opRead + } + return data +} + +// ReadByte reads and returns the next byte from the buffer. +// If no byte is available, it returns error os.EOF. +func (b *Buffer) ReadByte() (c byte, err os.Error) { + b.lastRead = opInvalid + if b.off >= len(b.buf) { + // Buffer is empty, reset to recover space. + b.Truncate(0) + return 0, os.EOF + } + c = b.buf[b.off] + b.off++ + b.lastRead = opRead + return c, nil +} + +// ReadRune reads and returns the next UTF-8-encoded +// Unicode code point from the buffer. +// If no bytes are available, the error returned is os.EOF. +// If the bytes are an erroneous UTF-8 encoding, it +// consumes one byte and returns U+FFFD, 1. +func (b *Buffer) ReadRune() (r int, size int, err os.Error) { + b.lastRead = opInvalid + if b.off >= len(b.buf) { + // Buffer is empty, reset to recover space. + b.Truncate(0) + return 0, 0, os.EOF + } + b.lastRead = opReadRune + c := b.buf[b.off] + if c < utf8.RuneSelf { + b.off++ + return int(c), 1, nil + } + r, n := utf8.DecodeRune(b.buf[b.off:]) + b.off += n + return r, n, nil +} + +// UnreadRune unreads the last rune returned by ReadRune. +// If the most recent read or write operation on the buffer was +// not a ReadRune, UnreadRune returns an error. (In this regard +// it is stricter than UnreadByte, which will unread the last byte +// from any read operation.) +func (b *Buffer) UnreadRune() os.Error { + if b.lastRead != opReadRune { + return os.ErrorString("bytes.Buffer: UnreadRune: previous operation was not ReadRune") + } + b.lastRead = opInvalid + if b.off > 0 { + _, n := utf8.DecodeLastRune(b.buf[0:b.off]) + b.off -= n + } + return nil +} + +// UnreadByte unreads the last byte returned by the most recent +// read operation. If write has happened since the last read, UnreadByte +// returns an error. +func (b *Buffer) UnreadByte() os.Error { + if b.lastRead != opReadRune && b.lastRead != opRead { + return os.ErrorString("bytes.Buffer: UnreadByte: previous operation was not a read") + } + b.lastRead = opInvalid + if b.off > 0 { + b.off-- + } + return nil +} + +// NewBuffer creates and initializes a new Buffer using buf as its initial +// contents. It is intended to prepare a Buffer to read existing data. It +// can also be used to size the internal buffer for writing. To do that, +// buf should have the desired capacity but a length of zero. +func NewBuffer(buf []byte) *Buffer { return &Buffer{buf: buf} } + +// NewBufferString creates and initializes a new Buffer using string s as its +// initial contents. It is intended to prepare a buffer to read an existing +// string. +func NewBufferString(s string) *Buffer { + return &Buffer{buf: []byte(s)} +} diff --git a/libgo/go/bytes/buffer_test.go b/libgo/go/bytes/buffer_test.go new file mode 100644 index 000000000..509793d24 --- /dev/null +++ b/libgo/go/bytes/buffer_test.go @@ -0,0 +1,349 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bytes_test + +import ( + . "bytes" + "rand" + "testing" + "utf8" +) + + +const N = 10000 // make this bigger for a larger (and slower) test +var data string // test data for write tests +var bytes []byte // test data; same as data but as a slice. + + +func init() { + bytes = make([]byte, N) + for i := 0; i < N; i++ { + bytes[i] = 'a' + byte(i%26) + } + data = string(bytes) +} + +// Verify that contents of buf match the string s. +func check(t *testing.T, testname string, buf *Buffer, s string) { + bytes := buf.Bytes() + str := buf.String() + if buf.Len() != len(bytes) { + t.Errorf("%s: buf.Len() == %d, len(buf.Bytes()) == %d", testname, buf.Len(), len(bytes)) + } + + if buf.Len() != len(str) { + t.Errorf("%s: buf.Len() == %d, len(buf.String()) == %d", testname, buf.Len(), len(str)) + } + + if buf.Len() != len(s) { + t.Errorf("%s: buf.Len() == %d, len(s) == %d", testname, buf.Len(), len(s)) + } + + if string(bytes) != s { + t.Errorf("%s: string(buf.Bytes()) == %q, s == %q", testname, string(bytes), s) + } +} + + +// Fill buf through n writes of string fus. +// The initial contents of buf corresponds to the string s; +// the result is the final contents of buf returned as a string. +func fillString(t *testing.T, testname string, buf *Buffer, s string, n int, fus string) string { + check(t, testname+" (fill 1)", buf, s) + for ; n > 0; n-- { + m, err := buf.WriteString(fus) + if m != len(fus) { + t.Errorf(testname+" (fill 2): m == %d, expected %d", m, len(fus)) + } + if err != nil { + t.Errorf(testname+" (fill 3): err should always be nil, found err == %s", err) + } + s += fus + check(t, testname+" (fill 4)", buf, s) + } + return s +} + + +// Fill buf through n writes of byte slice fub. +// The initial contents of buf corresponds to the string s; +// the result is the final contents of buf returned as a string. +func fillBytes(t *testing.T, testname string, buf *Buffer, s string, n int, fub []byte) string { + check(t, testname+" (fill 1)", buf, s) + for ; n > 0; n-- { + m, err := buf.Write(fub) + if m != len(fub) { + t.Errorf(testname+" (fill 2): m == %d, expected %d", m, len(fub)) + } + if err != nil { + t.Errorf(testname+" (fill 3): err should always be nil, found err == %s", err) + } + s += string(fub) + check(t, testname+" (fill 4)", buf, s) + } + return s +} + + +func TestNewBuffer(t *testing.T) { + buf := NewBuffer(bytes) + check(t, "NewBuffer", buf, data) +} + + +func TestNewBufferString(t *testing.T) { + buf := NewBufferString(data) + check(t, "NewBufferString", buf, data) +} + + +// Empty buf through repeated reads into fub. +// The initial contents of buf corresponds to the string s. +func empty(t *testing.T, testname string, buf *Buffer, s string, fub []byte) { + check(t, testname+" (empty 1)", buf, s) + + for { + n, err := buf.Read(fub) + if n == 0 { + break + } + if err != nil { + t.Errorf(testname+" (empty 2): err should always be nil, found err == %s", err) + } + s = s[n:] + check(t, testname+" (empty 3)", buf, s) + } + + check(t, testname+" (empty 4)", buf, "") +} + + +func TestBasicOperations(t *testing.T) { + var buf Buffer + + for i := 0; i < 5; i++ { + check(t, "TestBasicOperations (1)", &buf, "") + + buf.Reset() + check(t, "TestBasicOperations (2)", &buf, "") + + buf.Truncate(0) + check(t, "TestBasicOperations (3)", &buf, "") + + n, err := buf.Write([]byte(data[0:1])) + if n != 1 { + t.Errorf("wrote 1 byte, but n == %d", n) + } + if err != nil { + t.Errorf("err should always be nil, but err == %s", err) + } + check(t, "TestBasicOperations (4)", &buf, "a") + + buf.WriteByte(data[1]) + check(t, "TestBasicOperations (5)", &buf, "ab") + + n, err = buf.Write([]byte(data[2:26])) + if n != 24 { + t.Errorf("wrote 25 bytes, but n == %d", n) + } + check(t, "TestBasicOperations (6)", &buf, string(data[0:26])) + + buf.Truncate(26) + check(t, "TestBasicOperations (7)", &buf, string(data[0:26])) + + buf.Truncate(20) + check(t, "TestBasicOperations (8)", &buf, string(data[0:20])) + + empty(t, "TestBasicOperations (9)", &buf, string(data[0:20]), make([]byte, 5)) + empty(t, "TestBasicOperations (10)", &buf, "", make([]byte, 100)) + + buf.WriteByte(data[1]) + c, err := buf.ReadByte() + if err != nil { + t.Error("ReadByte unexpected eof") + } + if c != data[1] { + t.Errorf("ReadByte wrong value c=%v", c) + } + c, err = buf.ReadByte() + if err == nil { + t.Error("ReadByte unexpected not eof") + } + } +} + + +func TestLargeStringWrites(t *testing.T) { + var buf Buffer + for i := 3; i < 30; i += 3 { + s := fillString(t, "TestLargeWrites (1)", &buf, "", 5, data) + empty(t, "TestLargeStringWrites (2)", &buf, s, make([]byte, len(data)/i)) + } + check(t, "TestLargeStringWrites (3)", &buf, "") +} + + +func TestLargeByteWrites(t *testing.T) { + var buf Buffer + for i := 3; i < 30; i += 3 { + s := fillBytes(t, "TestLargeWrites (1)", &buf, "", 5, bytes) + empty(t, "TestLargeByteWrites (2)", &buf, s, make([]byte, len(data)/i)) + } + check(t, "TestLargeByteWrites (3)", &buf, "") +} + + +func TestLargeStringReads(t *testing.T) { + var buf Buffer + for i := 3; i < 30; i += 3 { + s := fillString(t, "TestLargeReads (1)", &buf, "", 5, data[0:len(data)/i]) + empty(t, "TestLargeReads (2)", &buf, s, make([]byte, len(data))) + } + check(t, "TestLargeStringReads (3)", &buf, "") +} + + +func TestLargeByteReads(t *testing.T) { + var buf Buffer + for i := 3; i < 30; i += 3 { + s := fillBytes(t, "TestLargeReads (1)", &buf, "", 5, bytes[0:len(bytes)/i]) + empty(t, "TestLargeReads (2)", &buf, s, make([]byte, len(data))) + } + check(t, "TestLargeByteReads (3)", &buf, "") +} + + +func TestMixedReadsAndWrites(t *testing.T) { + var buf Buffer + s := "" + for i := 0; i < 50; i++ { + wlen := rand.Intn(len(data)) + if i%2 == 0 { + s = fillString(t, "TestMixedReadsAndWrites (1)", &buf, s, 1, data[0:wlen]) + } else { + s = fillBytes(t, "TestMixedReadsAndWrites (1)", &buf, s, 1, bytes[0:wlen]) + } + + rlen := rand.Intn(len(data)) + fub := make([]byte, rlen) + n, _ := buf.Read(fub) + s = s[n:] + } + empty(t, "TestMixedReadsAndWrites (2)", &buf, s, make([]byte, buf.Len())) +} + + +func TestNil(t *testing.T) { + var b *Buffer + if b.String() != "" { + t.Errorf("expcted ; got %q", b.String()) + } +} + + +func TestReadFrom(t *testing.T) { + var buf Buffer + for i := 3; i < 30; i += 3 { + s := fillBytes(t, "TestReadFrom (1)", &buf, "", 5, bytes[0:len(bytes)/i]) + var b Buffer + b.ReadFrom(&buf) + empty(t, "TestReadFrom (2)", &b, s, make([]byte, len(data))) + } +} + + +func TestWriteTo(t *testing.T) { + var buf Buffer + for i := 3; i < 30; i += 3 { + s := fillBytes(t, "TestReadFrom (1)", &buf, "", 5, bytes[0:len(bytes)/i]) + var b Buffer + buf.WriteTo(&b) + empty(t, "TestReadFrom (2)", &b, s, make([]byte, len(data))) + } +} + + +func TestRuneIO(t *testing.T) { + const NRune = 1000 + // Built a test array while we write the data + b := make([]byte, utf8.UTFMax*NRune) + var buf Buffer + n := 0 + for r := 0; r < NRune; r++ { + size := utf8.EncodeRune(b[n:], r) + nbytes, err := buf.WriteRune(r) + if err != nil { + t.Fatalf("WriteRune(%U) error: %s", r, err) + } + if nbytes != size { + t.Fatalf("WriteRune(%U) expected %d, got %d", r, size, nbytes) + } + n += size + } + b = b[0:n] + + // Check the resulting bytes + if !Equal(buf.Bytes(), b) { + t.Fatalf("incorrect result from WriteRune: %q not %q", buf.Bytes(), b) + } + + p := make([]byte, utf8.UTFMax) + // Read it back with ReadRune + for r := 0; r < NRune; r++ { + size := utf8.EncodeRune(p, r) + nr, nbytes, err := buf.ReadRune() + if nr != r || nbytes != size || err != nil { + t.Fatalf("ReadRune(%U) got %U,%d not %U,%d (err=%s)", r, nr, nbytes, r, size, err) + } + } + + // Check that UnreadRune works + buf.Reset() + buf.Write(b) + for r := 0; r < NRune; r++ { + r1, size, _ := buf.ReadRune() + if err := buf.UnreadRune(); err != nil { + t.Fatalf("UnreadRune(%U) got error %q", r, err) + } + r2, nbytes, err := buf.ReadRune() + if r1 != r2 || r1 != r || nbytes != size || err != nil { + t.Fatalf("ReadRune(%U) after UnreadRune got %U,%d not %U,%d (err=%s)", r, r2, nbytes, r, size, err) + } + } +} + + +func TestNext(t *testing.T) { + b := []byte{0, 1, 2, 3, 4} + tmp := make([]byte, 5) + for i := 0; i <= 5; i++ { + for j := i; j <= 5; j++ { + for k := 0; k <= 6; k++ { + // 0 <= i <= j <= 5; 0 <= k <= 6 + // Check that if we start with a buffer + // of length j at offset i and ask for + // Next(k), we get the right bytes. + buf := NewBuffer(b[0:j]) + n, _ := buf.Read(tmp[0:i]) + if n != i { + t.Fatalf("Read %d returned %d", i, n) + } + bb := buf.Next(k) + want := k + if want > j-i { + want = j - i + } + if len(bb) != want { + t.Fatalf("in %d,%d: len(Next(%d)) == %d", i, j, k, len(bb)) + } + for l, v := range bb { + if v != byte(l+i) { + t.Fatalf("in %d,%d: Next(%d)[%d] = %d, want %d", i, j, k, l, v, l+i) + } + } + } + } + } +} diff --git a/libgo/go/bytes/bytes.go b/libgo/go/bytes/bytes.go new file mode 100644 index 000000000..bfe2ef39d --- /dev/null +++ b/libgo/go/bytes/bytes.go @@ -0,0 +1,602 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The bytes package implements functions for the manipulation of byte slices. +// Analogous to the facilities of the strings package. +package bytes + +import ( + "unicode" + "utf8" +) + +// Compare returns an integer comparing the two byte arrays lexicographically. +// The result will be 0 if a==b, -1 if a < b, and +1 if a > b +func Compare(a, b []byte) int { + m := len(a) + if m > len(b) { + m = len(b) + } + for i, ac := range a[0:m] { + bc := b[i] + switch { + case ac > bc: + return 1 + case ac < bc: + return -1 + } + } + switch { + case len(a) < len(b): + return -1 + case len(a) > len(b): + return 1 + } + return 0 +} + +// Equal returns a boolean reporting whether a == b. +func Equal(a, b []byte) bool { + if len(a) != len(b) { + return false + } + for i, c := range a { + if c != b[i] { + return false + } + } + return true +} + +// explode splits s into an array of UTF-8 sequences, one per Unicode character (still arrays of bytes), +// up to a maximum of n byte arrays. Invalid UTF-8 sequences are chopped into individual bytes. +func explode(s []byte, n int) [][]byte { + if n <= 0 { + n = len(s) + } + a := make([][]byte, n) + var size int + na := 0 + for len(s) > 0 { + if na+1 >= n { + a[na] = s + na++ + break + } + _, size = utf8.DecodeRune(s) + a[na] = s[0:size] + s = s[size:] + na++ + } + return a[0:na] +} + +// Count counts the number of non-overlapping instances of sep in s. +func Count(s, sep []byte) int { + if len(sep) == 0 { + return utf8.RuneCount(s) + 1 + } + c := sep[0] + n := 0 + for i := 0; i+len(sep) <= len(s); i++ { + if s[i] == c && (len(sep) == 1 || Equal(s[i:i+len(sep)], sep)) { + n++ + i += len(sep) - 1 + } + } + return n +} + +// Index returns the index of the first instance of sep in s, or -1 if sep is not present in s. +func Index(s, sep []byte) int { + n := len(sep) + if n == 0 { + return 0 + } + c := sep[0] + for i := 0; i+n <= len(s); i++ { + if s[i] == c && (n == 1 || Equal(s[i:i+n], sep)) { + return i + } + } + return -1 +} + +func indexBytePortable(s []byte, c byte) int { + for i, b := range s { + if b == c { + return i + } + } + return -1 +} + +// LastIndex returns the index of the last instance of sep in s, or -1 if sep is not present in s. +func LastIndex(s, sep []byte) int { + n := len(sep) + if n == 0 { + return len(s) + } + c := sep[0] + for i := len(s) - n; i >= 0; i-- { + if s[i] == c && (n == 1 || Equal(s[i:i+n], sep)) { + return i + } + } + return -1 +} + +// IndexRune interprets s as a sequence of UTF-8-encoded Unicode code points. +// It returns the byte index of the first occurrence in s of the given rune. +// It returns -1 if rune is not present in s. +func IndexRune(s []byte, rune int) int { + for i := 0; i < len(s); { + r, size := utf8.DecodeRune(s[i:]) + if r == rune { + return i + } + i += size + } + return -1 +} + +// IndexAny interprets s as a sequence of UTF-8-encoded Unicode code points. +// It returns the byte index of the first occurrence in s of any of the Unicode +// code points in chars. It returns -1 if chars is empty or if there is no code +// point in common. +func IndexAny(s []byte, chars string) int { + if len(chars) > 0 { + var rune, width int + for i := 0; i < len(s); i += width { + rune = int(s[i]) + if rune < utf8.RuneSelf { + width = 1 + } else { + rune, width = utf8.DecodeRune(s[i:]) + } + for _, r := range chars { + if rune == r { + return i + } + } + } + } + return -1 +} + +// LastIndexAny interprets s as a sequence of UTF-8-encoded Unicode code +// points. It returns the byte index of the last occurrence in s of any of +// the Unicode code points in chars. It returns -1 if chars is empty or if +// there is no code point in common. +func LastIndexAny(s []byte, chars string) int { + if len(chars) > 0 { + for i := len(s); i > 0; { + rune, size := utf8.DecodeLastRune(s[0:i]) + i -= size + for _, m := range chars { + if rune == m { + return i + } + } + } + } + return -1 +} + +// Generic split: splits after each instance of sep, +// including sepSave bytes of sep in the subarrays. +func genSplit(s, sep []byte, sepSave, n int) [][]byte { + if n == 0 { + return nil + } + if len(sep) == 0 { + return explode(s, n) + } + if n < 0 { + n = Count(s, sep) + 1 + } + c := sep[0] + start := 0 + a := make([][]byte, n) + na := 0 + for i := 0; i+len(sep) <= len(s) && na+1 < n; i++ { + if s[i] == c && (len(sep) == 1 || Equal(s[i:i+len(sep)], sep)) { + a[na] = s[start : i+sepSave] + na++ + start = i + len(sep) + i += len(sep) - 1 + } + } + a[na] = s[start:] + return a[0 : na+1] +} + +// Split slices s into subslices separated by sep and returns a slice of +// the subslices between those separators. +// If sep is empty, Split splits after each UTF-8 sequence. +// The count determines the number of subslices to return: +// n > 0: at most n subslices; the last subslice will be the unsplit remainder. +// n == 0: the result is nil (zero subslices) +// n < 0: all subslices +func Split(s, sep []byte, n int) [][]byte { return genSplit(s, sep, 0, n) } + +// SplitAfter slices s into subslices after each instance of sep and +// returns a slice of those subslices. +// If sep is empty, Split splits after each UTF-8 sequence. +// The count determines the number of subslices to return: +// n > 0: at most n subslices; the last subslice will be the unsplit remainder. +// n == 0: the result is nil (zero subslices) +// n < 0: all subslices +func SplitAfter(s, sep []byte, n int) [][]byte { + return genSplit(s, sep, len(sep), n) +} + +// Fields splits the array s around each instance of one or more consecutive white space +// characters, returning a slice of subarrays of s or an empty list if s contains only white space. +func Fields(s []byte) [][]byte { + return FieldsFunc(s, unicode.IsSpace) +} + +// FieldsFunc interprets s as a sequence of UTF-8-encoded Unicode code points. +// It splits the array s at each run of code points c satisfying f(c) and +// returns a slice of subarrays of s. If no code points in s satisfy f(c), an +// empty slice is returned. +func FieldsFunc(s []byte, f func(int) bool) [][]byte { + n := 0 + inField := false + for i := 0; i < len(s); { + rune, size := utf8.DecodeRune(s[i:]) + wasInField := inField + inField = !f(rune) + if inField && !wasInField { + n++ + } + i += size + } + + a := make([][]byte, n) + na := 0 + fieldStart := -1 + for i := 0; i <= len(s) && na < n; { + rune, size := utf8.DecodeRune(s[i:]) + if fieldStart < 0 && size > 0 && !f(rune) { + fieldStart = i + i += size + continue + } + if fieldStart >= 0 && (size == 0 || f(rune)) { + a[na] = s[fieldStart:i] + na++ + fieldStart = -1 + } + if size == 0 { + break + } + i += size + } + return a[0:na] +} + +// Join concatenates the elements of a to create a single byte array. The separator +// sep is placed between elements in the resulting array. +func Join(a [][]byte, sep []byte) []byte { + if len(a) == 0 { + return []byte{} + } + if len(a) == 1 { + return a[0] + } + n := len(sep) * (len(a) - 1) + for i := 0; i < len(a); i++ { + n += len(a[i]) + } + + b := make([]byte, n) + bp := 0 + for i := 0; i < len(a); i++ { + s := a[i] + for j := 0; j < len(s); j++ { + b[bp] = s[j] + bp++ + } + if i+1 < len(a) { + s = sep + for j := 0; j < len(s); j++ { + b[bp] = s[j] + bp++ + } + } + } + return b +} + +// HasPrefix tests whether the byte array s begins with prefix. +func HasPrefix(s, prefix []byte) bool { + return len(s) >= len(prefix) && Equal(s[0:len(prefix)], prefix) +} + +// HasSuffix tests whether the byte array s ends with suffix. +func HasSuffix(s, suffix []byte) bool { + return len(s) >= len(suffix) && Equal(s[len(s)-len(suffix):], suffix) +} + +// Map returns a copy of the byte array s with all its characters modified +// according to the mapping function. If mapping returns a negative value, the character is +// dropped from the string with no replacement. The characters in s and the +// output are interpreted as UTF-8-encoded Unicode code points. +func Map(mapping func(rune int) int, s []byte) []byte { + // In the worst case, the array can grow when mapped, making + // things unpleasant. But it's so rare we barge in assuming it's + // fine. It could also shrink but that falls out naturally. + maxbytes := len(s) // length of b + nbytes := 0 // number of bytes encoded in b + b := make([]byte, maxbytes) + for i := 0; i < len(s); { + wid := 1 + rune := int(s[i]) + if rune >= utf8.RuneSelf { + rune, wid = utf8.DecodeRune(s[i:]) + } + rune = mapping(rune) + if rune >= 0 { + if nbytes+utf8.RuneLen(rune) > maxbytes { + // Grow the buffer. + maxbytes = maxbytes*2 + utf8.UTFMax + nb := make([]byte, maxbytes) + copy(nb, b[0:nbytes]) + b = nb + } + nbytes += utf8.EncodeRune(b[nbytes:maxbytes], rune) + } + i += wid + } + return b[0:nbytes] +} + +// Repeat returns a new byte slice consisting of count copies of b. +func Repeat(b []byte, count int) []byte { + nb := make([]byte, len(b)*count) + bp := 0 + for i := 0; i < count; i++ { + for j := 0; j < len(b); j++ { + nb[bp] = b[j] + bp++ + } + } + return nb +} + +// ToUpper returns a copy of the byte array s with all Unicode letters mapped to their upper case. +func ToUpper(s []byte) []byte { return Map(unicode.ToUpper, s) } + +// ToUpper returns a copy of the byte array s with all Unicode letters mapped to their lower case. +func ToLower(s []byte) []byte { return Map(unicode.ToLower, s) } + +// ToTitle returns a copy of the byte array s with all Unicode letters mapped to their title case. +func ToTitle(s []byte) []byte { return Map(unicode.ToTitle, s) } + +// ToUpperSpecial returns a copy of the byte array s with all Unicode letters mapped to their +// upper case, giving priority to the special casing rules. +func ToUpperSpecial(_case unicode.SpecialCase, s []byte) []byte { + return Map(func(r int) int { return _case.ToUpper(r) }, s) +} + +// ToLowerSpecial returns a copy of the byte array s with all Unicode letters mapped to their +// lower case, giving priority to the special casing rules. +func ToLowerSpecial(_case unicode.SpecialCase, s []byte) []byte { + return Map(func(r int) int { return _case.ToLower(r) }, s) +} + +// ToTitleSpecial returns a copy of the byte array s with all Unicode letters mapped to their +// title case, giving priority to the special casing rules. +func ToTitleSpecial(_case unicode.SpecialCase, s []byte) []byte { + return Map(func(r int) int { return _case.ToTitle(r) }, s) +} + + +// isSeparator reports whether the rune could mark a word boundary. +// TODO: update when package unicode captures more of the properties. +func isSeparator(rune int) bool { + // ASCII alphanumerics and underscore are not separators + if rune <= 0x7F { + switch { + case '0' <= rune && rune <= '9': + return false + case 'a' <= rune && rune <= 'z': + return false + case 'A' <= rune && rune <= 'Z': + return false + case rune == '_': + return false + } + return true + } + // Letters and digits are not separators + if unicode.IsLetter(rune) || unicode.IsDigit(rune) { + return false + } + // Otherwise, all we can do for now is treat spaces as separators. + return unicode.IsSpace(rune) +} + +// BUG(r): The rule Title uses for word boundaries does not handle Unicode punctuation properly. + +// Title returns a copy of s with all Unicode letters that begin words +// mapped to their title case. +func Title(s []byte) []byte { + // Use a closure here to remember state. + // Hackish but effective. Depends on Map scanning in order and calling + // the closure once per rune. + prev := ' ' + return Map( + func(r int) int { + if isSeparator(prev) { + prev = r + return unicode.ToTitle(r) + } + prev = r + return r + }, + s) +} + +// TrimLeftFunc returns a subslice of s by slicing off all leading UTF-8-encoded +// Unicode code points c that satisfy f(c). +func TrimLeftFunc(s []byte, f func(r int) bool) []byte { + i := indexFunc(s, f, false) + if i == -1 { + return nil + } + return s[i:] +} + +// TrimRightFunc returns a subslice of s by slicing off all trailing UTF-8 +// encoded Unicode code points c that satisfy f(c). +func TrimRightFunc(s []byte, f func(r int) bool) []byte { + i := lastIndexFunc(s, f, false) + if i >= 0 && s[i] >= utf8.RuneSelf { + _, wid := utf8.DecodeRune(s[i:]) + i += wid + } else { + i++ + } + return s[0:i] +} + +// TrimFunc returns a subslice of s by slicing off all leading and trailing +// UTF-8-encoded Unicode code points c that satisfy f(c). +func TrimFunc(s []byte, f func(r int) bool) []byte { + return TrimRightFunc(TrimLeftFunc(s, f), f) +} + +// IndexFunc interprets s as a sequence of UTF-8-encoded Unicode code points. +// It returns the byte index in s of the first Unicode +// code point satisfying f(c), or -1 if none do. +func IndexFunc(s []byte, f func(r int) bool) int { + return indexFunc(s, f, true) +} + +// LastIndexFunc interprets s as a sequence of UTF-8-encoded Unicode code points. +// It returns the byte index in s of the last Unicode +// code point satisfying f(c), or -1 if none do. +func LastIndexFunc(s []byte, f func(r int) bool) int { + return lastIndexFunc(s, f, true) +} + +// indexFunc is the same as IndexFunc except that if +// truth==false, the sense of the predicate function is +// inverted. +func indexFunc(s []byte, f func(r int) bool, truth bool) int { + start := 0 + for start < len(s) { + wid := 1 + rune := int(s[start]) + if rune >= utf8.RuneSelf { + rune, wid = utf8.DecodeRune(s[start:]) + } + if f(rune) == truth { + return start + } + start += wid + } + return -1 +} + +// lastIndexFunc is the same as LastIndexFunc except that if +// truth==false, the sense of the predicate function is +// inverted. +func lastIndexFunc(s []byte, f func(r int) bool, truth bool) int { + for i := len(s); i > 0; { + rune, size := utf8.DecodeLastRune(s[0:i]) + i -= size + if f(rune) == truth { + return i + } + } + return -1 +} + +func makeCutsetFunc(cutset string) func(rune int) bool { + return func(rune int) bool { + for _, c := range cutset { + if c == rune { + return true + } + } + return false + } +} + +// Trim returns a subslice of s by slicing off all leading and +// trailing UTF-8-encoded Unicode code points contained in cutset. +func Trim(s []byte, cutset string) []byte { + return TrimFunc(s, makeCutsetFunc(cutset)) +} + +// TrimLeft returns a subslice of s by slicing off all leading +// UTF-8-encoded Unicode code points contained in cutset. +func TrimLeft(s []byte, cutset string) []byte { + return TrimLeftFunc(s, makeCutsetFunc(cutset)) +} + +// TrimRight returns a subslice of s by slicing off all trailing +// UTF-8-encoded Unicode code points that are contained in cutset. +func TrimRight(s []byte, cutset string) []byte { + return TrimRightFunc(s, makeCutsetFunc(cutset)) +} + +// TrimSpace returns a subslice of s by slicing off all leading and +// trailing white space, as defined by Unicode. +func TrimSpace(s []byte) []byte { + return TrimFunc(s, unicode.IsSpace) +} + +// Runes returns a slice of runes (Unicode code points) equivalent to s. +func Runes(s []byte) []int { + t := make([]int, utf8.RuneCount(s)) + i := 0 + for len(s) > 0 { + r, l := utf8.DecodeRune(s) + t[i] = r + i++ + s = s[l:] + } + return t +} + +// Replace returns a copy of the slice s with the first n +// non-overlapping instances of old replaced by new. +// If n < 0, there is no limit on the number of replacements. +func Replace(s, old, new []byte, n int) []byte { + if n == 0 { + return s // avoid allocation + } + // Compute number of replacements. + if m := Count(s, old); m == 0 { + return s // avoid allocation + } else if n <= 0 || m < n { + n = m + } + + // Apply replacements to buffer. + t := make([]byte, len(s)+n*(len(new)-len(old))) + w := 0 + start := 0 + for i := 0; i < n; i++ { + j := start + if len(old) == 0 { + if i > 0 { + _, wid := utf8.DecodeRune(s[start:]) + j += wid + } + } else { + j += Index(s[start:], old) + } + w += copy(t[w:], s[start:j]) + w += copy(t[w:], new) + start = j + len(old) + } + w += copy(t[w:], s[start:]) + return t[0:w] +} diff --git a/libgo/go/bytes/bytes_decl.go b/libgo/go/bytes/bytes_decl.go new file mode 100644 index 000000000..5d2b9e639 --- /dev/null +++ b/libgo/go/bytes/bytes_decl.go @@ -0,0 +1,8 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bytes + +// IndexByte returns the index of the first instance of c in s, or -1 if c is not present in s. +func IndexByte(s []byte, c byte) int // asm_$GOARCH.s diff --git a/libgo/go/bytes/bytes_test.go b/libgo/go/bytes/bytes_test.go new file mode 100644 index 000000000..063686ec5 --- /dev/null +++ b/libgo/go/bytes/bytes_test.go @@ -0,0 +1,844 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bytes_test + +import ( + . "bytes" + "testing" + "unicode" + "utf8" +) + +func eq(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := 0; i < len(a); i++ { + if a[i] != b[i] { + return false + } + } + return true +} + +func arrayOfString(a [][]byte) []string { + result := make([]string, len(a)) + for j := 0; j < len(a); j++ { + result[j] = string(a[j]) + } + return result +} + +// For ease of reading, the test cases use strings that are converted to byte +// arrays before invoking the functions. + +var abcd = "abcd" +var faces = "☺☻☹" +var commas = "1,2,3,4" +var dots = "1....2....3....4" + +type BinOpTest struct { + a string + b string + i int +} + +var comparetests = []BinOpTest{ + {"", "", 0}, + {"a", "", 1}, + {"", "a", -1}, + {"abc", "abc", 0}, + {"ab", "abc", -1}, + {"abc", "ab", 1}, + {"x", "ab", 1}, + {"ab", "x", -1}, + {"x", "a", 1}, + {"b", "x", -1}, +} + +func TestCompare(t *testing.T) { + for _, tt := range comparetests { + a := []byte(tt.a) + b := []byte(tt.b) + cmp := Compare(a, b) + eql := Equal(a, b) + if cmp != tt.i { + t.Errorf(`Compare(%q, %q) = %v`, tt.a, tt.b, cmp) + } + if eql != (tt.i == 0) { + t.Errorf(`Equal(%q, %q) = %v`, tt.a, tt.b, eql) + } + } +} + +var indexTests = []BinOpTest{ + {"", "", 0}, + {"", "a", -1}, + {"", "foo", -1}, + {"fo", "foo", -1}, + {"foo", "foo", 0}, + {"oofofoofooo", "f", 2}, + {"oofofoofooo", "foo", 4}, + {"barfoobarfoo", "foo", 3}, + {"foo", "", 0}, + {"foo", "o", 1}, + {"abcABCabc", "A", 3}, + // cases with one byte strings - test IndexByte and special case in Index() + {"", "a", -1}, + {"x", "a", -1}, + {"x", "x", 0}, + {"abc", "a", 0}, + {"abc", "b", 1}, + {"abc", "c", 2}, + {"abc", "x", -1}, + {"barfoobarfooyyyzzzyyyzzzyyyzzzyyyxxxzzzyyy", "x", 33}, + {"foofyfoobarfoobar", "y", 4}, + {"oooooooooooooooooooooo", "r", -1}, +} + +var lastIndexTests = []BinOpTest{ + {"", "", 0}, + {"", "a", -1}, + {"", "foo", -1}, + {"fo", "foo", -1}, + {"foo", "foo", 0}, + {"foo", "f", 0}, + {"oofofoofooo", "f", 7}, + {"oofofoofooo", "foo", 7}, + {"barfoobarfoo", "foo", 9}, + {"foo", "", 3}, + {"foo", "o", 2}, + {"abcABCabc", "A", 3}, + {"abcABCabc", "a", 6}, +} + +var indexAnyTests = []BinOpTest{ + {"", "", -1}, + {"", "a", -1}, + {"", "abc", -1}, + {"a", "", -1}, + {"a", "a", 0}, + {"aaa", "a", 0}, + {"abc", "xyz", -1}, + {"abc", "xcz", 2}, + {"ab☺c", "x☺yz", 2}, + {"aRegExp*", ".(|)*+?^$[]", 7}, + {dots + dots + dots, " ", -1}, +} + +var lastIndexAnyTests = []BinOpTest{ + {"", "", -1}, + {"", "a", -1}, + {"", "abc", -1}, + {"a", "", -1}, + {"a", "a", 0}, + {"aaa", "a", 2}, + {"abc", "xyz", -1}, + {"abc", "ab", 1}, + {"a☺b☻c☹d", "uvw☻xyz", 2 + len("☺")}, + {"a.RegExp*", ".(|)*+?^$[]", 8}, + {dots + dots + dots, " ", -1}, +} + +var indexRuneTests = []BinOpTest{ + {"", "a", -1}, + {"", "☺", -1}, + {"foo", "☹", -1}, + {"foo", "o", 1}, + {"foo☺bar", "☺", 3}, + {"foo☺☻☹bar", "☹", 9}, +} + +// Execute f on each test case. funcName should be the name of f; it's used +// in failure reports. +func runIndexTests(t *testing.T, f func(s, sep []byte) int, funcName string, testCases []BinOpTest) { + for _, test := range testCases { + a := []byte(test.a) + b := []byte(test.b) + actual := f(a, b) + if actual != test.i { + t.Errorf("%s(%q,%q) = %v; want %v", funcName, a, b, actual, test.i) + } + } +} + +func runIndexAnyTests(t *testing.T, f func(s []byte, chars string) int, funcName string, testCases []BinOpTest) { + for _, test := range testCases { + a := []byte(test.a) + actual := f(a, test.b) + if actual != test.i { + t.Errorf("%s(%q,%q) = %v; want %v", funcName, a, test.b, actual, test.i) + } + } +} + +func TestIndex(t *testing.T) { runIndexTests(t, Index, "Index", indexTests) } +func TestLastIndex(t *testing.T) { runIndexTests(t, LastIndex, "LastIndex", lastIndexTests) } +func TestIndexAny(t *testing.T) { runIndexAnyTests(t, IndexAny, "IndexAny", indexAnyTests) } +func TestLastIndexAny(t *testing.T) { + runIndexAnyTests(t, LastIndexAny, "LastIndexAny", lastIndexAnyTests) +} + +func TestIndexByte(t *testing.T) { + for _, tt := range indexTests { + if len(tt.b) != 1 { + continue + } + a := []byte(tt.a) + b := tt.b[0] + pos := IndexByte(a, b) + if pos != tt.i { + t.Errorf(`IndexByte(%q, '%c') = %v`, tt.a, b, pos) + } + posp := IndexBytePortable(a, b) + if posp != tt.i { + t.Errorf(`indexBytePortable(%q, '%c') = %v`, tt.a, b, posp) + } + } +} + +// test a larger buffer with different sizes and alignments +func TestIndexByteBig(t *testing.T) { + const n = 1024 + b := make([]byte, n) + for i := 0; i < n; i++ { + // different start alignments + b1 := b[i:] + for j := 0; j < len(b1); j++ { + b1[j] = 'x' + pos := IndexByte(b1, 'x') + if pos != j { + t.Errorf("IndexByte(%q, 'x') = %v", b1, pos) + } + b1[j] = 0 + pos = IndexByte(b1, 'x') + if pos != -1 { + t.Errorf("IndexByte(%q, 'x') = %v", b1, pos) + } + } + // different end alignments + b1 = b[:i] + for j := 0; j < len(b1); j++ { + b1[j] = 'x' + pos := IndexByte(b1, 'x') + if pos != j { + t.Errorf("IndexByte(%q, 'x') = %v", b1, pos) + } + b1[j] = 0 + pos = IndexByte(b1, 'x') + if pos != -1 { + t.Errorf("IndexByte(%q, 'x') = %v", b1, pos) + } + } + // different start and end alignments + b1 = b[i/2 : n-(i+1)/2] + for j := 0; j < len(b1); j++ { + b1[j] = 'x' + pos := IndexByte(b1, 'x') + if pos != j { + t.Errorf("IndexByte(%q, 'x') = %v", b1, pos) + } + b1[j] = 0 + pos = IndexByte(b1, 'x') + if pos != -1 { + t.Errorf("IndexByte(%q, 'x') = %v", b1, pos) + } + } + } +} + +func TestIndexRune(t *testing.T) { + for _, tt := range indexRuneTests { + a := []byte(tt.a) + r, _ := utf8.DecodeRuneInString(tt.b) + pos := IndexRune(a, r) + if pos != tt.i { + t.Errorf(`IndexRune(%q, '%c') = %v`, tt.a, r, pos) + } + } +} + +func BenchmarkIndexByte4K(b *testing.B) { bmIndex(b, IndexByte, 4<<10) } + +func BenchmarkIndexByte4M(b *testing.B) { bmIndex(b, IndexByte, 4<<20) } + +func BenchmarkIndexByte64M(b *testing.B) { bmIndex(b, IndexByte, 64<<20) } + +func BenchmarkIndexBytePortable4K(b *testing.B) { + bmIndex(b, IndexBytePortable, 4<<10) +} + +func BenchmarkIndexBytePortable4M(b *testing.B) { + bmIndex(b, IndexBytePortable, 4<<20) +} + +func BenchmarkIndexBytePortable64M(b *testing.B) { + bmIndex(b, IndexBytePortable, 64<<20) +} + +var bmbuf []byte + +func bmIndex(b *testing.B, index func([]byte, byte) int, n int) { + if len(bmbuf) < n { + bmbuf = make([]byte, n) + } + b.SetBytes(int64(n)) + buf := bmbuf[0:n] + buf[n-1] = 'x' + for i := 0; i < b.N; i++ { + j := index(buf, 'x') + if j != n-1 { + println("bad index", j) + panic("bad index") + } + } + buf[n-1] = '0' +} + +type ExplodeTest struct { + s string + n int + a []string +} + +var explodetests = []ExplodeTest{ + {"", -1, []string{}}, + {abcd, -1, []string{"a", "b", "c", "d"}}, + {faces, -1, []string{"☺", "☻", "☹"}}, + {abcd, 2, []string{"a", "bcd"}}, +} + +func TestExplode(t *testing.T) { + for _, tt := range explodetests { + a := Split([]byte(tt.s), nil, tt.n) + result := arrayOfString(a) + if !eq(result, tt.a) { + t.Errorf(`Explode("%s", %d) = %v; want %v`, tt.s, tt.n, result, tt.a) + continue + } + s := Join(a, []byte{}) + if string(s) != tt.s { + t.Errorf(`Join(Explode("%s", %d), "") = "%s"`, tt.s, tt.n, s) + } + } +} + + +type SplitTest struct { + s string + sep string + n int + a []string +} + +var splittests = []SplitTest{ + {abcd, "a", 0, nil}, + {abcd, "a", -1, []string{"", "bcd"}}, + {abcd, "z", -1, []string{"abcd"}}, + {abcd, "", -1, []string{"a", "b", "c", "d"}}, + {commas, ",", -1, []string{"1", "2", "3", "4"}}, + {dots, "...", -1, []string{"1", ".2", ".3", ".4"}}, + {faces, "☹", -1, []string{"☺☻", ""}}, + {faces, "~", -1, []string{faces}}, + {faces, "", -1, []string{"☺", "☻", "☹"}}, + {"1 2 3 4", " ", 3, []string{"1", "2", "3 4"}}, + {"1 2", " ", 3, []string{"1", "2"}}, + {"123", "", 2, []string{"1", "23"}}, + {"123", "", 17, []string{"1", "2", "3"}}, +} + +func TestSplit(t *testing.T) { + for _, tt := range splittests { + a := Split([]byte(tt.s), []byte(tt.sep), tt.n) + result := arrayOfString(a) + if !eq(result, tt.a) { + t.Errorf(`Split(%q, %q, %d) = %v; want %v`, tt.s, tt.sep, tt.n, result, tt.a) + continue + } + if tt.n == 0 { + continue + } + s := Join(a, []byte(tt.sep)) + if string(s) != tt.s { + t.Errorf(`Join(Split(%q, %q, %d), %q) = %q`, tt.s, tt.sep, tt.n, tt.sep, s) + } + } +} + +var splitaftertests = []SplitTest{ + {abcd, "a", -1, []string{"a", "bcd"}}, + {abcd, "z", -1, []string{"abcd"}}, + {abcd, "", -1, []string{"a", "b", "c", "d"}}, + {commas, ",", -1, []string{"1,", "2,", "3,", "4"}}, + {dots, "...", -1, []string{"1...", ".2...", ".3...", ".4"}}, + {faces, "☹", -1, []string{"☺☻☹", ""}}, + {faces, "~", -1, []string{faces}}, + {faces, "", -1, []string{"☺", "☻", "☹"}}, + {"1 2 3 4", " ", 3, []string{"1 ", "2 ", "3 4"}}, + {"1 2 3", " ", 3, []string{"1 ", "2 ", "3"}}, + {"1 2", " ", 3, []string{"1 ", "2"}}, + {"123", "", 2, []string{"1", "23"}}, + {"123", "", 17, []string{"1", "2", "3"}}, +} + +func TestSplitAfter(t *testing.T) { + for _, tt := range splitaftertests { + a := SplitAfter([]byte(tt.s), []byte(tt.sep), tt.n) + result := arrayOfString(a) + if !eq(result, tt.a) { + t.Errorf(`Split(%q, %q, %d) = %v; want %v`, tt.s, tt.sep, tt.n, result, tt.a) + continue + } + s := Join(a, nil) + if string(s) != tt.s { + t.Errorf(`Join(Split(%q, %q, %d), %q) = %q`, tt.s, tt.sep, tt.n, tt.sep, s) + } + } +} + +type FieldsTest struct { + s string + a []string +} + +var fieldstests = []FieldsTest{ + {"", []string{}}, + {" ", []string{}}, + {" \t ", []string{}}, + {" abc ", []string{"abc"}}, + {"1 2 3 4", []string{"1", "2", "3", "4"}}, + {"1 2 3 4", []string{"1", "2", "3", "4"}}, + {"1\t\t2\t\t3\t4", []string{"1", "2", "3", "4"}}, + {"1\u20002\u20013\u20024", []string{"1", "2", "3", "4"}}, + {"\u2000\u2001\u2002", []string{}}, + {"\n™\t™\n", []string{"™", "™"}}, + {faces, []string{faces}}, +} + +func TestFields(t *testing.T) { + for _, tt := range fieldstests { + a := Fields([]byte(tt.s)) + result := arrayOfString(a) + if !eq(result, tt.a) { + t.Errorf("Fields(%q) = %v; want %v", tt.s, a, tt.a) + continue + } + } +} + +func TestFieldsFunc(t *testing.T) { + pred := func(c int) bool { return c == 'X' } + var fieldsFuncTests = []FieldsTest{ + {"", []string{}}, + {"XX", []string{}}, + {"XXhiXXX", []string{"hi"}}, + {"aXXbXXXcX", []string{"a", "b", "c"}}, + } + for _, tt := range fieldsFuncTests { + a := FieldsFunc([]byte(tt.s), pred) + result := arrayOfString(a) + if !eq(result, tt.a) { + t.Errorf("FieldsFunc(%q) = %v, want %v", tt.s, a, tt.a) + } + } +} + +// Test case for any function which accepts and returns a byte array. +// For ease of creation, we write the byte arrays as strings. +type StringTest struct { + in, out string +} + +var upperTests = []StringTest{ + {"", ""}, + {"abc", "ABC"}, + {"AbC123", "ABC123"}, + {"azAZ09_", "AZAZ09_"}, + {"\u0250\u0250\u0250\u0250\u0250", "\u2C6F\u2C6F\u2C6F\u2C6F\u2C6F"}, // grows one byte per char +} + +var lowerTests = []StringTest{ + {"", ""}, + {"abc", "abc"}, + {"AbC123", "abc123"}, + {"azAZ09_", "azaz09_"}, + {"\u2C6D\u2C6D\u2C6D\u2C6D\u2C6D", "\u0251\u0251\u0251\u0251\u0251"}, // shrinks one byte per char +} + +const space = "\t\v\r\f\n\u0085\u00a0\u2000\u3000" + +var trimSpaceTests = []StringTest{ + {"", ""}, + {"abc", "abc"}, + {space + "abc" + space, "abc"}, + {" ", ""}, + {" \t\r\n \t\t\r\r\n\n ", ""}, + {" \t\r\n x\t\t\r\r\n\n ", "x"}, + {" \u2000\t\r\n x\t\t\r\r\ny\n \u3000", "x\t\t\r\r\ny"}, + {"1 \t\r\n2", "1 \t\r\n2"}, + {" x\x80", "x\x80"}, + {" x\xc0", "x\xc0"}, + {"x \xc0\xc0 ", "x \xc0\xc0"}, + {"x \xc0", "x \xc0"}, + {"x \xc0 ", "x \xc0"}, + {"x \xc0\xc0 ", "x \xc0\xc0"}, + {"x ☺\xc0\xc0 ", "x ☺\xc0\xc0"}, + {"x ☺ ", "x ☺"}, +} + +// Execute f on each test case. funcName should be the name of f; it's used +// in failure reports. +func runStringTests(t *testing.T, f func([]byte) []byte, funcName string, testCases []StringTest) { + for _, tc := range testCases { + actual := string(f([]byte(tc.in))) + if actual != tc.out { + t.Errorf("%s(%q) = %q; want %q", funcName, tc.in, actual, tc.out) + } + } +} + +func tenRunes(rune int) string { + r := make([]int, 10) + for i := range r { + r[i] = rune + } + return string(r) +} + +// User-defined self-inverse mapping function +func rot13(rune int) int { + step := 13 + if rune >= 'a' && rune <= 'z' { + return ((rune - 'a' + step) % 26) + 'a' + } + if rune >= 'A' && rune <= 'Z' { + return ((rune - 'A' + step) % 26) + 'A' + } + return rune +} + +func TestMap(t *testing.T) { + // Run a couple of awful growth/shrinkage tests + a := tenRunes('a') + + // 1. Grow. This triggers two reallocations in Map. + maxRune := func(rune int) int { return unicode.MaxRune } + m := Map(maxRune, []byte(a)) + expect := tenRunes(unicode.MaxRune) + if string(m) != expect { + t.Errorf("growing: expected %q got %q", expect, m) + } + + // 2. Shrink + minRune := func(rune int) int { return 'a' } + m = Map(minRune, []byte(tenRunes(unicode.MaxRune))) + expect = a + if string(m) != expect { + t.Errorf("shrinking: expected %q got %q", expect, m) + } + + // 3. Rot13 + m = Map(rot13, []byte("a to zed")) + expect = "n gb mrq" + if string(m) != expect { + t.Errorf("rot13: expected %q got %q", expect, m) + } + + // 4. Rot13^2 + m = Map(rot13, Map(rot13, []byte("a to zed"))) + expect = "a to zed" + if string(m) != expect { + t.Errorf("rot13: expected %q got %q", expect, m) + } + + // 5. Drop + dropNotLatin := func(rune int) int { + if unicode.Is(unicode.Latin, rune) { + return rune + } + return -1 + } + m = Map(dropNotLatin, []byte("Hello, 세계")) + expect = "Hello" + if string(m) != expect { + t.Errorf("drop: expected %q got %q", expect, m) + } +} + +func TestToUpper(t *testing.T) { runStringTests(t, ToUpper, "ToUpper", upperTests) } + +func TestToLower(t *testing.T) { runStringTests(t, ToLower, "ToLower", lowerTests) } + +func TestTrimSpace(t *testing.T) { runStringTests(t, TrimSpace, "TrimSpace", trimSpaceTests) } + +type RepeatTest struct { + in, out string + count int +} + +var RepeatTests = []RepeatTest{ + {"", "", 0}, + {"", "", 1}, + {"", "", 2}, + {"-", "", 0}, + {"-", "-", 1}, + {"-", "----------", 10}, + {"abc ", "abc abc abc ", 3}, +} + +func TestRepeat(t *testing.T) { + for _, tt := range RepeatTests { + tin := []byte(tt.in) + tout := []byte(tt.out) + a := Repeat(tin, tt.count) + if !Equal(a, tout) { + t.Errorf("Repeat(%q, %d) = %q; want %q", tin, tt.count, a, tout) + continue + } + } +} + +func runesEqual(a, b []int) bool { + if len(a) != len(b) { + return false + } + for i, r := range a { + if r != b[i] { + return false + } + } + return true +} + +type RunesTest struct { + in string + out []int + lossy bool +} + +var RunesTests = []RunesTest{ + {"", []int{}, false}, + {" ", []int{32}, false}, + {"ABC", []int{65, 66, 67}, false}, + {"abc", []int{97, 98, 99}, false}, + {"\u65e5\u672c\u8a9e", []int{26085, 26412, 35486}, false}, + {"ab\x80c", []int{97, 98, 0xFFFD, 99}, true}, + {"ab\xc0c", []int{97, 98, 0xFFFD, 99}, true}, +} + +func TestRunes(t *testing.T) { + for _, tt := range RunesTests { + tin := []byte(tt.in) + a := Runes(tin) + if !runesEqual(a, tt.out) { + t.Errorf("Runes(%q) = %v; want %v", tin, a, tt.out) + continue + } + if !tt.lossy { + // can only test reassembly if we didn't lose information + s := string(a) + if s != tt.in { + t.Errorf("string(Runes(%q)) = %x; want %x", tin, s, tin) + } + } + } +} + + +type TrimTest struct { + f func([]byte, string) []byte + in, cutset, out string +} + +var trimTests = []TrimTest{ + {Trim, "abba", "a", "bb"}, + {Trim, "abba", "ab", ""}, + {TrimLeft, "abba", "ab", ""}, + {TrimRight, "abba", "ab", ""}, + {TrimLeft, "abba", "a", "bba"}, + {TrimRight, "abba", "a", "abb"}, + {Trim, "", "<>", "tag"}, + {Trim, "* listitem", " *", "listitem"}, + {Trim, `"quote"`, `"`, "quote"}, + {Trim, "\u2C6F\u2C6F\u0250\u0250\u2C6F\u2C6F", "\u2C6F", "\u0250\u0250"}, + //empty string tests + {Trim, "abba", "", "abba"}, + {Trim, "", "123", ""}, + {Trim, "", "", ""}, + {TrimLeft, "abba", "", "abba"}, + {TrimLeft, "", "123", ""}, + {TrimLeft, "", "", ""}, + {TrimRight, "abba", "", "abba"}, + {TrimRight, "", "123", ""}, + {TrimRight, "", "", ""}, + {TrimRight, "☺\xc0", "☺", "☺\xc0"}, +} + +func TestTrim(t *testing.T) { + for _, tc := range trimTests { + actual := string(tc.f([]byte(tc.in), tc.cutset)) + var name string + switch tc.f { + case Trim: + name = "Trim" + case TrimLeft: + name = "TrimLeft" + case TrimRight: + name = "TrimRight" + default: + t.Error("Undefined trim function") + } + if actual != tc.out { + t.Errorf("%s(%q, %q) = %q; want %q", name, tc.in, tc.cutset, actual, tc.out) + } + } +} + +type predicate struct { + f func(r int) bool + name string +} + +var isSpace = predicate{unicode.IsSpace, "IsSpace"} +var isDigit = predicate{unicode.IsDigit, "IsDigit"} +var isUpper = predicate{unicode.IsUpper, "IsUpper"} +var isValidRune = predicate{ + func(r int) bool { + return r != utf8.RuneError + }, + "IsValidRune", +} + +type TrimFuncTest struct { + f predicate + in, out string +} + +func not(p predicate) predicate { + return predicate{ + func(r int) bool { + return !p.f(r) + }, + "not " + p.name, + } +} + +var trimFuncTests = []TrimFuncTest{ + {isSpace, space + " hello " + space, "hello"}, + {isDigit, "\u0e50\u0e5212hello34\u0e50\u0e51", "hello"}, + {isUpper, "\u2C6F\u2C6F\u2C6F\u2C6FABCDhelloEF\u2C6F\u2C6FGH\u2C6F\u2C6F", "hello"}, + {not(isSpace), "hello" + space + "hello", space}, + {not(isDigit), "hello\u0e50\u0e521234\u0e50\u0e51helo", "\u0e50\u0e521234\u0e50\u0e51"}, + {isValidRune, "ab\xc0a\xc0cd", "\xc0a\xc0"}, + {not(isValidRune), "\xc0a\xc0", "a"}, +} + +func TestTrimFunc(t *testing.T) { + for _, tc := range trimFuncTests { + actual := string(TrimFunc([]byte(tc.in), tc.f.f)) + if actual != tc.out { + t.Errorf("TrimFunc(%q, %q) = %q; want %q", tc.in, tc.f.name, actual, tc.out) + } + } +} + +type IndexFuncTest struct { + in string + f predicate + first, last int +} + +var indexFuncTests = []IndexFuncTest{ + {"", isValidRune, -1, -1}, + {"abc", isDigit, -1, -1}, + {"0123", isDigit, 0, 3}, + {"a1b", isDigit, 1, 1}, + {space, isSpace, 0, len(space) - 3}, // last rune in space is 3 bytes + {"\u0e50\u0e5212hello34\u0e50\u0e51", isDigit, 0, 18}, + {"\u2C6F\u2C6F\u2C6F\u2C6FABCDhelloEF\u2C6F\u2C6FGH\u2C6F\u2C6F", isUpper, 0, 34}, + {"12\u0e50\u0e52hello34\u0e50\u0e51", not(isDigit), 8, 12}, + + // tests of invalid UTF-8 + {"\x801", isDigit, 1, 1}, + {"\x80abc", isDigit, -1, -1}, + {"\xc0a\xc0", isValidRune, 1, 1}, + {"\xc0a\xc0", not(isValidRune), 0, 2}, + {"\xc0☺\xc0", not(isValidRune), 0, 4}, + {"\xc0☺\xc0\xc0", not(isValidRune), 0, 5}, + {"ab\xc0a\xc0cd", not(isValidRune), 2, 4}, + {"a\xe0\x80cd", not(isValidRune), 1, 2}, +} + +func TestIndexFunc(t *testing.T) { + for _, tc := range indexFuncTests { + first := IndexFunc([]byte(tc.in), tc.f.f) + if first != tc.first { + t.Errorf("IndexFunc(%q, %s) = %d; want %d", tc.in, tc.f.name, first, tc.first) + } + last := LastIndexFunc([]byte(tc.in), tc.f.f) + if last != tc.last { + t.Errorf("LastIndexFunc(%q, %s) = %d; want %d", tc.in, tc.f.name, last, tc.last) + } + } +} + +type ReplaceTest struct { + in string + old, new string + n int + out string +} + +var ReplaceTests = []ReplaceTest{ + {"hello", "l", "L", 0, "hello"}, + {"hello", "l", "L", -1, "heLLo"}, + {"hello", "x", "X", -1, "hello"}, + {"", "x", "X", -1, ""}, + {"radar", "r", "", -1, "ada"}, + {"", "", "<>", -1, "<>"}, + {"banana", "a", "<>", -1, "b<>n<>n<>"}, + {"banana", "a", "<>", 1, "b<>nana"}, + {"banana", "a", "<>", 1000, "b<>n<>n<>"}, + {"banana", "an", "<>", -1, "b<><>a"}, + {"banana", "ana", "<>", -1, "b<>na"}, + {"banana", "", "<>", -1, "<>b<>a<>n<>a<>n<>a<>"}, + {"banana", "", "<>", 10, "<>b<>a<>n<>a<>n<>a<>"}, + {"banana", "", "<>", 6, "<>b<>a<>n<>a<>n<>a"}, + {"banana", "", "<>", 5, "<>b<>a<>n<>a<>na"}, + {"banana", "", "<>", 1, "<>banana"}, + {"banana", "a", "a", -1, "banana"}, + {"banana", "a", "a", 1, "banana"}, + {"☺☻☹", "", "<>", -1, "<>☺<>☻<>☹<>"}, +} + +func TestReplace(t *testing.T) { + for _, tt := range ReplaceTests { + if s := string(Replace([]byte(tt.in), []byte(tt.old), []byte(tt.new), tt.n)); s != tt.out { + t.Errorf("Replace(%q, %q, %q, %d) = %q, want %q", tt.in, tt.old, tt.new, tt.n, s, tt.out) + } + } +} + +type TitleTest struct { + in, out string +} + +var TitleTests = []TitleTest{ + {"", ""}, + {"a", "A"}, + {" aaa aaa aaa ", " Aaa Aaa Aaa "}, + {" Aaa Aaa Aaa ", " Aaa Aaa Aaa "}, + {"123a456", "123a456"}, + {"double-blind", "Double-Blind"}, + {"ÿøû", "Ÿøû"}, +} + +func TestTitle(t *testing.T) { + for _, tt := range TitleTests { + if s := string(Title([]byte(tt.in))); s != tt.out { + t.Errorf("Title(%q) = %q, want %q", tt.in, s, tt.out) + } + } +} diff --git a/libgo/go/bytes/export_test.go b/libgo/go/bytes/export_test.go new file mode 100644 index 000000000..b65428d9c --- /dev/null +++ b/libgo/go/bytes/export_test.go @@ -0,0 +1,8 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bytes + +// Export func for testing +var IndexBytePortable = indexBytePortable diff --git a/libgo/go/bytes/indexbyte.c b/libgo/go/bytes/indexbyte.c new file mode 100644 index 000000000..a0a963e93 --- /dev/null +++ b/libgo/go/bytes/indexbyte.c @@ -0,0 +1,28 @@ +/* indexbyte.c -- implement bytes.IndexByte for Go. + + Copyright 2009 The Go Authors. All rights reserved. + Use of this source code is governed by a BSD-style + license that can be found in the LICENSE file. */ + +#include + +#include "array.h" + +/* This is in C so that the compiler can optimize it appropriately. + We deliberately don't split the stack in case it does call the + library function, which shouldn't need much stack space. */ + +int IndexByte (struct __go_open_array, char) + asm ("libgo_bytes.bytes.IndexByte") + __attribute__ ((no_split_stack)); + +int +IndexByte (struct __go_open_array s, char b) +{ + char *p; + + p = __builtin_memchr (s.__values, b, s.__count); + if (p == NULL) + return -1; + return p - (char *) s.__values; +} diff --git a/libgo/go/cmath/abs.go b/libgo/go/cmath/abs.go new file mode 100644 index 000000000..725dc4e98 --- /dev/null +++ b/libgo/go/cmath/abs.go @@ -0,0 +1,12 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The cmath package provides basic constants +// and mathematical functions for complex numbers. +package cmath + +import "math" + +// Abs returns the absolute value (also called the modulus) of x. +func Abs(x complex128) float64 { return math.Hypot(real(x), imag(x)) } diff --git a/libgo/go/cmath/asin.go b/libgo/go/cmath/asin.go new file mode 100644 index 000000000..d6a3ca480 --- /dev/null +++ b/libgo/go/cmath/asin.go @@ -0,0 +1,170 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cmath + +import "math" + +// The original C code, the long comment, and the constants +// below are from http://netlib.sandia.gov/cephes/c9x-complex/clog.c. +// The go code is a simplified version of the original C. +// +// Cephes Math Library Release 2.8: June, 2000 +// Copyright 1984, 1987, 1989, 1992, 2000 by Stephen L. Moshier +// +// The readme file at http://netlib.sandia.gov/cephes/ says: +// Some software in this archive may be from the book _Methods and +// Programs for Mathematical Functions_ (Prentice-Hall or Simon & Schuster +// International, 1989) or from the Cephes Mathematical Library, a +// commercial product. In either event, it is copyrighted by the author. +// What you see here may be used freely but it comes with no support or +// guarantee. +// +// The two known misprints in the book are repaired here in the +// source listings for the gamma function and the incomplete beta +// integral. +// +// Stephen L. Moshier +// moshier@na-net.ornl.gov + +// Complex circular arc sine +// +// DESCRIPTION: +// +// Inverse complex sine: +// 2 +// w = -i clog( iz + csqrt( 1 - z ) ). +// +// casin(z) = -i casinh(iz) +// +// ACCURACY: +// +// Relative error: +// arithmetic domain # trials peak rms +// DEC -10,+10 10100 2.1e-15 3.4e-16 +// IEEE -10,+10 30000 2.2e-14 2.7e-15 +// Larger relative error can be observed for z near zero. +// Also tested by csin(casin(z)) = z. + +// Asin returns the inverse sine of x. +func Asin(x complex128) complex128 { + if imag(x) == 0 { + if math.Fabs(real(x)) > 1 { + return complex(math.Pi/2, 0) // DOMAIN error + } + return complex(math.Asin(real(x)), 0) + } + ct := complex(-imag(x), real(x)) // i * x + xx := x * x + x1 := complex(1-real(xx), -imag(xx)) // 1 - x*x + x2 := Sqrt(x1) // x2 = sqrt(1 - x*x) + w := Log(ct + x2) + return complex(imag(w), -real(w)) // -i * w +} + +// Asinh returns the inverse hyperbolic sine of x. +func Asinh(x complex128) complex128 { + // TODO check range + if imag(x) == 0 { + if math.Fabs(real(x)) > 1 { + return complex(math.Pi/2, 0) // DOMAIN error + } + return complex(math.Asinh(real(x)), 0) + } + xx := x * x + x1 := complex(1+real(xx), imag(xx)) // 1 + x*x + return Log(x + Sqrt(x1)) // log(x + sqrt(1 + x*x)) +} + +// Complex circular arc cosine +// +// DESCRIPTION: +// +// w = arccos z = PI/2 - arcsin z. +// +// ACCURACY: +// +// Relative error: +// arithmetic domain # trials peak rms +// DEC -10,+10 5200 1.6e-15 2.8e-16 +// IEEE -10,+10 30000 1.8e-14 2.2e-15 + +// Acos returns the inverse cosine of x. +func Acos(x complex128) complex128 { + w := Asin(x) + return complex(math.Pi/2-real(w), -imag(w)) +} + +// Acosh returns the inverse hyperbolic cosine of x. +func Acosh(x complex128) complex128 { + w := Acos(x) + if imag(w) <= 0 { + return complex(-imag(w), real(w)) // i * w + } + return complex(imag(w), -real(w)) // -i * w +} + +// Complex circular arc tangent +// +// DESCRIPTION: +// +// If +// z = x + iy, +// +// then +// 1 ( 2x ) +// Re w = - arctan(-----------) + k PI +// 2 ( 2 2) +// (1 - x - y ) +// +// ( 2 2) +// 1 (x + (y+1) ) +// Im w = - log(------------) +// 4 ( 2 2) +// (x + (y-1) ) +// +// Where k is an arbitrary integer. +// +// catan(z) = -i catanh(iz). +// +// ACCURACY: +// +// Relative error: +// arithmetic domain # trials peak rms +// DEC -10,+10 5900 1.3e-16 7.8e-18 +// IEEE -10,+10 30000 2.3e-15 8.5e-17 +// The check catan( ctan(z) ) = z, with |x| and |y| < PI/2, +// had peak relative error 1.5e-16, rms relative error +// 2.9e-17. See also clog(). + +// Atan returns the inverse tangent of x. +func Atan(x complex128) complex128 { + if real(x) == 0 && imag(x) > 1 { + return NaN() + } + + x2 := real(x) * real(x) + a := 1 - x2 - imag(x)*imag(x) + if a == 0 { + return NaN() + } + t := 0.5 * math.Atan2(2*real(x), a) + w := reducePi(t) + + t = imag(x) - 1 + b := x2 + t*t + if b == 0 { + return NaN() + } + t = imag(x) + 1 + c := (x2 + t*t) / b + return complex(w, 0.25*math.Log(c)) +} + +// Atanh returns the inverse hyperbolic tangent of x. +func Atanh(x complex128) complex128 { + z := complex(-imag(x), real(x)) // z = i * x + z = Atan(z) + return complex(imag(z), -real(z)) // z = -i * z +} diff --git a/libgo/go/cmath/cmath_test.go b/libgo/go/cmath/cmath_test.go new file mode 100644 index 000000000..6a595b0a6 --- /dev/null +++ b/libgo/go/cmath/cmath_test.go @@ -0,0 +1,853 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cmath + +import ( + "math" + "testing" +) + +var vc26 = []complex128{ + (4.97901192488367350108546816 + 7.73887247457810456552351752i), + (7.73887247457810456552351752 - 0.27688005719200159404635997i), + (-0.27688005719200159404635997 - 5.01060361827107492160848778i), + (-5.01060361827107492160848778 + 9.63629370719841737980004837i), + (9.63629370719841737980004837 + 2.92637723924396464525443662i), + (2.92637723924396464525443662 + 5.22908343145930665230025625i), + (5.22908343145930665230025625 + 2.72793991043601025126008608i), + (2.72793991043601025126008608 + 1.82530809168085506044576505i), + (1.82530809168085506044576505 - 8.68592476857560136238589621i), + (-8.68592476857560136238589621 + 4.97901192488367350108546816i), +} +var vc = []complex128{ + (4.9790119248836735e+00 + 7.7388724745781045e+00i), + (7.7388724745781045e+00 - 2.7688005719200159e-01i), + (-2.7688005719200159e-01 - 5.0106036182710749e+00i), + (-5.0106036182710749e+00 + 9.6362937071984173e+00i), + (9.6362937071984173e+00 + 2.9263772392439646e+00i), + (2.9263772392439646e+00 + 5.2290834314593066e+00i), + (5.2290834314593066e+00 + 2.7279399104360102e+00i), + (2.7279399104360102e+00 + 1.8253080916808550e+00i), + (1.8253080916808550e+00 - 8.6859247685756013e+00i), + (-8.6859247685756013e+00 + 4.9790119248836735e+00i), +} + +// The expected results below were computed by the high precision calculators +// at http://keisan.casio.com/. More exact input values (array vc[], above) +// were obtained by printing them with "%.26f". The answers were calculated +// to 26 digits (by using the "Digit number" drop-down control of each +// calculator). + +var abs = []float64{ + 9.2022120669932650313380972e+00, + 7.7438239742296106616261394e+00, + 5.0182478202557746902556648e+00, + 1.0861137372799545160704002e+01, + 1.0070841084922199607011905e+01, + 5.9922447613166942183705192e+00, + 5.8978784056736762299945176e+00, + 3.2822866700678709020367184e+00, + 8.8756430028990417290744307e+00, + 1.0011785496777731986390856e+01, +} + +var acos = []complex128{ + (1.0017679804707456328694569 - 2.9138232718554953784519807i), + (0.03606427612041407369636057 + 2.7358584434576260925091256i), + (1.6249365462333796703711823 + 2.3159537454335901187730929i), + (2.0485650849650740120660391 - 3.0795576791204117911123886i), + (0.29621132089073067282488147 - 3.0007392508200622519398814i), + (1.0664555914934156601503632 - 2.4872865024796011364747111i), + (0.48681307452231387690013905 - 2.463655912283054555225301i), + (0.6116977071277574248407752 - 1.8734458851737055262693056i), + (1.3649311280370181331184214 + 2.8793528632328795424123832i), + (2.6189310485682988308904501 - 2.9956543302898767795858704i), +} +var acosh = []complex128{ + (2.9138232718554953784519807 + 1.0017679804707456328694569i), + (2.7358584434576260925091256 - 0.03606427612041407369636057i), + (2.3159537454335901187730929 - 1.6249365462333796703711823i), + (3.0795576791204117911123886 + 2.0485650849650740120660391i), + (3.0007392508200622519398814 + 0.29621132089073067282488147i), + (2.4872865024796011364747111 + 1.0664555914934156601503632i), + (2.463655912283054555225301 + 0.48681307452231387690013905i), + (1.8734458851737055262693056 + 0.6116977071277574248407752i), + (2.8793528632328795424123832 - 1.3649311280370181331184214i), + (2.9956543302898767795858704 + 2.6189310485682988308904501i), +} +var asin = []complex128{ + (0.56902834632415098636186476 + 2.9138232718554953784519807i), + (1.5347320506744825455349611 - 2.7358584434576260925091256i), + (-0.054140219438483051139860579 - 2.3159537454335901187730929i), + (-0.47776875817017739283471738 + 3.0795576791204117911123886i), + (1.2745850059041659464064402 + 3.0007392508200622519398814i), + (0.50434073530148095908095852 + 2.4872865024796011364747111i), + (1.0839832522725827423311826 + 2.463655912283054555225301i), + (0.9590986196671391943905465 + 1.8734458851737055262693056i), + (0.20586519875787848611290031 - 2.8793528632328795424123832i), + (-1.0481347217734022116591284 + 2.9956543302898767795858704i), +} +var asinh = []complex128{ + (2.9113760469415295679342185 + 0.99639459545704326759805893i), + (2.7441755423994259061579029 - 0.035468308789000500601119392i), + (-2.2962136462520690506126678 - 1.5144663565690151885726707i), + (-3.0771233459295725965402455 + 1.0895577967194013849422294i), + (3.0048366100923647417557027 + 0.29346979169819220036454168i), + (2.4800059370795363157364643 + 1.0545868606049165710424232i), + (2.4718773838309585611141821 + 0.47502344364250803363708842i), + (1.8910743588080159144378396 + 0.56882925572563602341139174i), + (2.8735426423367341878069406 - 1.362376149648891420997548i), + (-2.9981750586172477217567878 + 0.5183571985225367505624207i), +} +var atan = []complex128{ + (1.5115747079332741358607654 + 0.091324403603954494382276776i), + (1.4424504323482602560806727 - 0.0045416132642803911503770933i), + (-1.5593488703630532674484026 - 0.20163295409248362456446431i), + (-1.5280619472445889867794105 + 0.081721556230672003746956324i), + (1.4759909163240799678221039 + 0.028602969320691644358773586i), + (1.4877353772046548932715555 + 0.14566877153207281663773599i), + (1.4206983927779191889826 + 0.076830486127880702249439993i), + (1.3162236060498933364869556 + 0.16031313000467530644933363i), + (1.5473450684303703578810093 - 0.11064907507939082484935782i), + (-1.4841462340185253987375812 + 0.049341850305024399493142411i), +} +var atanh = []complex128{ + (0.058375027938968509064640438 + 1.4793488495105334458167782i), + (0.12977343497790381229915667 - 1.5661009410463561327262499i), + (-0.010576456067347252072200088 - 1.3743698658402284549750563i), + (-0.042218595678688358882784918 + 1.4891433968166405606692604i), + (0.095218997991316722061828397 + 1.5416884098777110330499698i), + (0.079965459366890323857556487 + 1.4252510353873192700350435i), + (0.15051245471980726221708301 + 1.4907432533016303804884461i), + (0.25082072933993987714470373 + 1.392057665392187516442986i), + (0.022896108815797135846276662 - 1.4609224989282864208963021i), + (-0.08665624101841876130537396 + 1.5207902036935093480142159i), +} +var conj = []complex128{ + (4.9790119248836735e+00 - 7.7388724745781045e+00i), + (7.7388724745781045e+00 + 2.7688005719200159e-01i), + (-2.7688005719200159e-01 + 5.0106036182710749e+00i), + (-5.0106036182710749e+00 - 9.6362937071984173e+00i), + (9.6362937071984173e+00 - 2.9263772392439646e+00i), + (2.9263772392439646e+00 - 5.2290834314593066e+00i), + (5.2290834314593066e+00 - 2.7279399104360102e+00i), + (2.7279399104360102e+00 - 1.8253080916808550e+00i), + (1.8253080916808550e+00 + 8.6859247685756013e+00i), + (-8.6859247685756013e+00 - 4.9790119248836735e+00i), +} +var cos = []complex128{ + (3.024540920601483938336569e+02 + 1.1073797572517071650045357e+03i), + (1.192858682649064973252758e-01 + 2.7857554122333065540970207e-01i), + (7.2144394304528306603857962e+01 - 2.0500129667076044169954205e+01i), + (2.24921952538403984190541e+03 - 7.317363745602773587049329e+03i), + (-9.148222970032421760015498e+00 + 1.953124661113563541862227e+00i), + (-9.116081175857732248227078e+01 - 1.992669213569952232487371e+01i), + (3.795639179042704640002918e+00 + 6.623513350981458399309662e+00i), + (-2.9144840732498869560679084e+00 - 1.214620271628002917638748e+00i), + (-7.45123482501299743872481e+02 + 2.8641692314488080814066734e+03i), + (-5.371977967039319076416747e+01 + 4.893348341339375830564624e+01i), +} +var cosh = []complex128{ + (8.34638383523018249366948e+00 + 7.2181057886425846415112064e+01i), + (1.10421967379919366952251e+03 - 3.1379638689277575379469861e+02i), + (3.051485206773701584738512e-01 - 2.6805384730105297848044485e-01i), + (-7.33294728684187933370938e+01 + 1.574445942284918251038144e+01i), + (-7.478643293945957535757355e+03 + 1.6348382209913353929473321e+03i), + (4.622316522966235701630926e+00 - 8.088695185566375256093098e+00i), + (-8.544333183278877406197712e+01 + 3.7505836120128166455231717e+01i), + (-1.934457815021493925115198e+00 + 7.3725859611767228178358673e+00i), + (-2.352958770061749348353548e+00 - 2.034982010440878358915409e+00i), + (7.79756457532134748165069e+02 + 2.8549350716819176560377717e+03i), +} +var exp = []complex128{ + (1.669197736864670815125146e+01 + 1.4436895109507663689174096e+02i), + (2.2084389286252583447276212e+03 - 6.2759289284909211238261917e+02i), + (2.227538273122775173434327e-01 + 7.2468284028334191250470034e-01i), + (-6.5182985958153548997881627e-03 - 1.39965837915193860879044e-03i), + (-1.4957286524084015746110777e+04 + 3.269676455931135688988042e+03i), + (9.218158701983105935659273e+00 - 1.6223985291084956009304582e+01i), + (-1.7088175716853040841444505e+02 + 7.501382609870410713795546e+01i), + (-3.852461315830959613132505e+00 + 1.4808420423156073221970892e+01i), + (-4.586775503301407379786695e+00 - 4.178501081246873415144744e+00i), + (4.451337963005453491095747e-05 - 1.62977574205442915935263e-04i), +} +var log = []complex128{ + (2.2194438972179194425697051e+00 + 9.9909115046919291062461269e-01i), + (2.0468956191154167256337289e+00 - 3.5762575021856971295156489e-02i), + (1.6130808329853860438751244e+00 - 1.6259990074019058442232221e+00i), + (2.3851910394823008710032651e+00 + 2.0502936359659111755031062e+00i), + (2.3096442270679923004800651e+00 + 2.9483213155446756211881774e-01i), + (1.7904660933974656106951860e+00 + 1.0605860367252556281902109e+00i), + (1.7745926939841751666177512e+00 + 4.8084556083358307819310911e-01i), + (1.1885403350045342425648780e+00 + 5.8969634164776659423195222e-01i), + (2.1833107837679082586772505e+00 - 1.3636647724582455028314573e+00i), + (2.3037629487273259170991671e+00 + 2.6210913895386013290915234e+00i), +} +var log10 = []complex128{ + (9.6389223745559042474184943e-01 + 4.338997735671419492599631e-01i), + (8.8895547241376579493490892e-01 - 1.5531488990643548254864806e-02i), + (7.0055210462945412305244578e-01 - 7.0616239649481243222248404e-01i), + (1.0358753067322445311676952e+00 + 8.9043121238134980156490909e-01i), + (1.003065742975330237172029e+00 + 1.2804396782187887479857811e-01i), + (7.7758954439739162532085157e-01 + 4.6060666333341810869055108e-01i), + (7.7069581462315327037689152e-01 + 2.0882857371769952195512475e-01i), + (5.1617650901191156135137239e-01 + 2.5610186717615977620363299e-01i), + (9.4819982567026639742663212e-01 - 5.9223208584446952284914289e-01i), + (1.0005115362454417135973429e+00 + 1.1383255270407412817250921e+00i), +} + +type ff struct { + r, theta float64 +} + +var polar = []ff{ + {9.2022120669932650313380972e+00, 9.9909115046919291062461269e-01}, + {7.7438239742296106616261394e+00, -3.5762575021856971295156489e-02}, + {5.0182478202557746902556648e+00, -1.6259990074019058442232221e+00}, + {1.0861137372799545160704002e+01, 2.0502936359659111755031062e+00}, + {1.0070841084922199607011905e+01, 2.9483213155446756211881774e-01}, + {5.9922447613166942183705192e+00, 1.0605860367252556281902109e+00}, + {5.8978784056736762299945176e+00, 4.8084556083358307819310911e-01}, + {3.2822866700678709020367184e+00, 5.8969634164776659423195222e-01}, + {8.8756430028990417290744307e+00, -1.3636647724582455028314573e+00}, + {1.0011785496777731986390856e+01, 2.6210913895386013290915234e+00}, +} +var pow = []complex128{ + (-2.499956739197529585028819e+00 + 1.759751724335650228957144e+00i), + (7.357094338218116311191939e+04 - 5.089973412479151648145882e+04i), + (1.320777296067768517259592e+01 - 3.165621914333901498921986e+01i), + (-3.123287828297300934072149e-07 - 1.9849567521490553032502223E-7i), + (8.0622651468477229614813e+04 - 7.80028727944573092944363e+04i), + (-1.0268824572103165858577141e+00 - 4.716844738244989776610672e-01i), + (-4.35953819012244175753187e+01 + 2.2036445974645306917648585e+02i), + (8.3556092283250594950239e-01 - 1.2261571947167240272593282e+01i), + (1.582292972120769306069625e+03 + 1.273564263524278244782512e+04i), + (6.592208301642122149025369e-08 + 2.584887236651661903526389e-08i), +} +var sin = []complex128{ + (-1.1073801774240233539648544e+03 + 3.024539773002502192425231e+02i), + (1.0317037521400759359744682e+00 - 3.2208979799929570242818e-02i), + (-2.0501952097271429804261058e+01 - 7.2137981348240798841800967e+01i), + (7.3173638080346338642193078e+03 + 2.249219506193664342566248e+03i), + (-1.964375633631808177565226e+00 - 9.0958264713870404464159683e+00i), + (1.992783647158514838337674e+01 - 9.11555769410191350416942e+01i), + (-6.680335650741921444300349e+00 + 3.763353833142432513086117e+00i), + (1.2794028166657459148245993e+00 - 2.7669092099795781155109602e+00i), + (2.8641693949535259594188879e+03 + 7.451234399649871202841615e+02i), + (-4.893811726244659135553033e+01 - 5.371469305562194635957655e+01i), +} +var sinh = []complex128{ + (8.34559353341652565758198e+00 + 7.2187893208650790476628899e+01i), + (1.1042192548260646752051112e+03 - 3.1379650595631635858792056e+02i), + (-8.239469336509264113041849e-02 + 9.9273668758439489098514519e-01i), + (7.332295456982297798219401e+01 - 1.574585908122833444899023e+01i), + (-7.4786432301380582103534216e+03 + 1.63483823493980029604071e+03i), + (4.595842179016870234028347e+00 - 8.135290105518580753211484e+00i), + (-8.543842533574163435246793e+01 + 3.750798997857594068272375e+01i), + (-1.918003500809465688017307e+00 + 7.4358344619793504041350251e+00i), + (-2.233816733239658031433147e+00 - 2.143519070805995056229335e+00i), + (-7.797564130187551181105341e+02 - 2.8549352346594918614806877e+03i), +} +var sqrt = []complex128{ + (2.6628203086086130543813948e+00 + 1.4531345674282185229796902e+00i), + (2.7823278427251986247149295e+00 - 4.9756907317005224529115567e-02i), + (1.5397025302089642757361015e+00 - 1.6271336573016637535695727e+00i), + (1.7103411581506875260277898e+00 + 2.8170677122737589676157029e+00i), + (3.1390392472953103383607947e+00 + 4.6612625849858653248980849e-01i), + (2.1117080764822417640789287e+00 + 1.2381170223514273234967850e+00i), + (2.3587032281672256703926939e+00 + 5.7827111903257349935720172e-01i), + (1.7335262588873410476661577e+00 + 5.2647258220721269141550382e-01i), + (2.3131094974708716531499282e+00 - 1.8775429304303785570775490e+00i), + (8.1420535745048086240947359e-01 + 3.0575897587277248522656113e+00i), +} +var tan = []complex128{ + (-1.928757919086441129134525e-07 + 1.0000003267499169073251826e+00i), + (1.242412685364183792138948e+00 - 3.17149693883133370106696e+00i), + (-4.6745126251587795225571826e-05 - 9.9992439225263959286114298e-01i), + (4.792363401193648192887116e-09 + 1.0000000070589333451557723e+00i), + (2.345740824080089140287315e-03 + 9.947733046570988661022763e-01i), + (-2.396030789494815566088809e-05 + 9.9994781345418591429826779e-01i), + (-7.370204836644931340905303e-03 + 1.0043553413417138987717748e+00i), + (-3.691803847992048527007457e-02 + 9.6475071993469548066328894e-01i), + (-2.781955256713729368401878e-08 - 1.000000049848910609006646e+00i), + (9.4281590064030478879791249e-05 + 9.9999119340863718183758545e-01i), +} +var tanh = []complex128{ + (1.0000921981225144748819918e+00 + 2.160986245871518020231507e-05i), + (9.9999967727531993209562591e-01 - 1.9953763222959658873657676e-07i), + (-1.765485739548037260789686e+00 + 1.7024216325552852445168471e+00i), + (-9.999189442732736452807108e-01 + 3.64906070494473701938098e-05i), + (9.9999999224622333738729767e-01 - 3.560088949517914774813046e-09i), + (1.0029324933367326862499343e+00 - 4.948790309797102353137528e-03i), + (9.9996113064788012488693567e-01 - 4.226995742097032481451259e-05i), + (1.0074784189316340029873945e+00 - 4.194050814891697808029407e-03i), + (9.9385534229718327109131502e-01 + 5.144217985914355502713437e-02i), + (-1.0000000491604982429364892e+00 - 2.901873195374433112227349e-08i), +} + +// special cases +var vcAbsSC = []complex128{ + NaN(), +} +var absSC = []float64{ + math.NaN(), +} +var vcAcosSC = []complex128{ + NaN(), +} +var acosSC = []complex128{ + NaN(), +} +var vcAcoshSC = []complex128{ + NaN(), +} +var acoshSC = []complex128{ + NaN(), +} +var vcAsinSC = []complex128{ + NaN(), +} +var asinSC = []complex128{ + NaN(), +} +var vcAsinhSC = []complex128{ + NaN(), +} +var asinhSC = []complex128{ + NaN(), +} +var vcAtanSC = []complex128{ + NaN(), +} +var atanSC = []complex128{ + NaN(), +} +var vcAtanhSC = []complex128{ + NaN(), +} +var atanhSC = []complex128{ + NaN(), +} +var vcConjSC = []complex128{ + NaN(), +} +var conjSC = []complex128{ + NaN(), +} +var vcCosSC = []complex128{ + NaN(), +} +var cosSC = []complex128{ + NaN(), +} +var vcCoshSC = []complex128{ + NaN(), +} +var coshSC = []complex128{ + NaN(), +} +var vcExpSC = []complex128{ + NaN(), +} +var expSC = []complex128{ + NaN(), +} +var vcIsNaNSC = []complex128{ + complex(math.Inf(-1), math.Inf(-1)), + complex(math.Inf(-1), math.NaN()), + complex(math.NaN(), math.Inf(-1)), + complex(0, math.NaN()), + complex(math.NaN(), 0), + complex(math.Inf(1), math.Inf(1)), + complex(math.Inf(1), math.NaN()), + complex(math.NaN(), math.Inf(1)), + complex(math.NaN(), math.NaN()), +} +var isNaNSC = []bool{ + false, + false, + false, + true, + true, + false, + false, + false, + true, +} +var vcLogSC = []complex128{ + NaN(), +} +var logSC = []complex128{ + NaN(), +} +var vcLog10SC = []complex128{ + NaN(), +} +var log10SC = []complex128{ + NaN(), +} +var vcPolarSC = []complex128{ + NaN(), +} +var polarSC = []ff{ + {math.NaN(), math.NaN()}, +} +var vcPowSC = [][2]complex128{ + {NaN(), NaN()}, +} +var powSC = []complex128{ + NaN(), +} +var vcSinSC = []complex128{ + NaN(), +} +var sinSC = []complex128{ + NaN(), +} +var vcSinhSC = []complex128{ + NaN(), +} +var sinhSC = []complex128{ + NaN(), +} +var vcSqrtSC = []complex128{ + NaN(), +} +var sqrtSC = []complex128{ + NaN(), +} +var vcTanSC = []complex128{ + NaN(), +} +var tanSC = []complex128{ + NaN(), +} +var vcTanhSC = []complex128{ + NaN(), +} +var tanhSC = []complex128{ + NaN(), +} + +// functions borrowed from pkg/math/all_test.go +func tolerance(a, b, e float64) bool { + d := a - b + if d < 0 { + d = -d + } + + if a != 0 { + e = e * a + if e < 0 { + e = -e + } + } + return d < e +} +func soclose(a, b, e float64) bool { return tolerance(a, b, e) } +func veryclose(a, b float64) bool { return tolerance(a, b, 4e-16) } +func alike(a, b float64) bool { + switch { + case a != a && b != b: // math.IsNaN(a) && math.IsNaN(b): + return true + case a == b: + return math.Signbit(a) == math.Signbit(b) + } + return false +} + +func cTolerance(a, b complex128, e float64) bool { + d := Abs(a - b) + if a != 0 { + e = e * Abs(a) + if e < 0 { + e = -e + } + } + return d < e +} +func cSoclose(a, b complex128, e float64) bool { return cTolerance(a, b, e) } +func cVeryclose(a, b complex128) bool { return cTolerance(a, b, 4e-16) } +func cAlike(a, b complex128) bool { + switch { + case IsNaN(a) && IsNaN(b): + return true + case a == b: + return math.Signbit(real(a)) == math.Signbit(real(b)) && math.Signbit(imag(a)) == math.Signbit(imag(b)) + } + return false +} + +func TestAbs(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Abs(vc[i]); !veryclose(abs[i], f) { + t.Errorf("Abs(%g) = %g, want %g", vc[i], f, abs[i]) + } + } + for i := 0; i < len(vcAbsSC); i++ { + if f := Abs(vcAbsSC[i]); !alike(absSC[i], f) { + t.Errorf("Abs(%g) = %g, want %g", vcAbsSC[i], f, absSC[i]) + } + } +} +func TestAcos(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Acos(vc[i]); !cSoclose(acos[i], f, 1e-14) { + t.Errorf("Acos(%g) = %g, want %g", vc[i], f, acos[i]) + } + } + for i := 0; i < len(vcAcosSC); i++ { + if f := Acos(vcAcosSC[i]); !cAlike(acosSC[i], f) { + t.Errorf("Acos(%g) = %g, want %g", vcAcosSC[i], f, acosSC[i]) + } + } +} +func TestAcosh(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Acosh(vc[i]); !cSoclose(acosh[i], f, 1e-14) { + t.Errorf("Acosh(%g) = %g, want %g", vc[i], f, acosh[i]) + } + } + for i := 0; i < len(vcAcoshSC); i++ { + if f := Acosh(vcAcoshSC[i]); !cAlike(acoshSC[i], f) { + t.Errorf("Acosh(%g) = %g, want %g", vcAcoshSC[i], f, acoshSC[i]) + } + } +} +func TestAsin(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Asin(vc[i]); !cSoclose(asin[i], f, 1e-14) { + t.Errorf("Asin(%g) = %g, want %g", vc[i], f, asin[i]) + } + } + for i := 0; i < len(vcAsinSC); i++ { + if f := Asin(vcAsinSC[i]); !cAlike(asinSC[i], f) { + t.Errorf("Asin(%g) = %g, want %g", vcAsinSC[i], f, asinSC[i]) + } + } +} +func TestAsinh(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Asinh(vc[i]); !cSoclose(asinh[i], f, 4e-15) { + t.Errorf("Asinh(%g) = %g, want %g", vc[i], f, asinh[i]) + } + } + for i := 0; i < len(vcAsinhSC); i++ { + if f := Asinh(vcAsinhSC[i]); !cAlike(asinhSC[i], f) { + t.Errorf("Asinh(%g) = %g, want %g", vcAsinhSC[i], f, asinhSC[i]) + } + } +} +func TestAtan(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Atan(vc[i]); !cVeryclose(atan[i], f) { + t.Errorf("Atan(%g) = %g, want %g", vc[i], f, atan[i]) + } + } + for i := 0; i < len(vcAtanSC); i++ { + if f := Atan(vcAtanSC[i]); !cAlike(atanSC[i], f) { + t.Errorf("Atan(%g) = %g, want %g", vcAtanSC[i], f, atanSC[i]) + } + } +} +func TestAtanh(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Atanh(vc[i]); !cVeryclose(atanh[i], f) { + t.Errorf("Atanh(%g) = %g, want %g", vc[i], f, atanh[i]) + } + } + for i := 0; i < len(vcAtanhSC); i++ { + if f := Atanh(vcAtanhSC[i]); !cAlike(atanhSC[i], f) { + t.Errorf("Atanh(%g) = %g, want %g", vcAtanhSC[i], f, atanhSC[i]) + } + } +} +func TestConj(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Conj(vc[i]); !cVeryclose(conj[i], f) { + t.Errorf("Conj(%g) = %g, want %g", vc[i], f, conj[i]) + } + } + for i := 0; i < len(vcConjSC); i++ { + if f := Conj(vcConjSC[i]); !cAlike(conjSC[i], f) { + t.Errorf("Conj(%g) = %g, want %g", vcConjSC[i], f, conjSC[i]) + } + } +} +func TestCos(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Cos(vc[i]); !cSoclose(cos[i], f, 3e-15) { + t.Errorf("Cos(%g) = %g, want %g", vc[i], f, cos[i]) + } + } + for i := 0; i < len(vcCosSC); i++ { + if f := Cos(vcCosSC[i]); !cAlike(cosSC[i], f) { + t.Errorf("Cos(%g) = %g, want %g", vcCosSC[i], f, cosSC[i]) + } + } +} +func TestCosh(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Cosh(vc[i]); !cSoclose(cosh[i], f, 2e-15) { + t.Errorf("Cosh(%g) = %g, want %g", vc[i], f, cosh[i]) + } + } + for i := 0; i < len(vcCoshSC); i++ { + if f := Cosh(vcCoshSC[i]); !cAlike(coshSC[i], f) { + t.Errorf("Cosh(%g) = %g, want %g", vcCoshSC[i], f, coshSC[i]) + } + } +} +func TestExp(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Exp(vc[i]); !cSoclose(exp[i], f, 1e-15) { + t.Errorf("Exp(%g) = %g, want %g", vc[i], f, exp[i]) + } + } + for i := 0; i < len(vcExpSC); i++ { + if f := Exp(vcExpSC[i]); !cAlike(expSC[i], f) { + t.Errorf("Exp(%g) = %g, want %g", vcExpSC[i], f, expSC[i]) + } + } +} +func TestIsNaN(t *testing.T) { + for i := 0; i < len(vcIsNaNSC); i++ { + if f := IsNaN(vcIsNaNSC[i]); isNaNSC[i] != f { + t.Errorf("IsNaN(%v) = %v, want %v", vcIsNaNSC[i], f, isNaNSC[i]) + } + } +} +func TestLog(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Log(vc[i]); !cVeryclose(log[i], f) { + t.Errorf("Log(%g) = %g, want %g", vc[i], f, log[i]) + } + } + for i := 0; i < len(vcLogSC); i++ { + if f := Log(vcLogSC[i]); !cAlike(logSC[i], f) { + t.Errorf("Log(%g) = %g, want %g", vcLogSC[i], f, logSC[i]) + } + } +} +func TestLog10(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Log10(vc[i]); !cVeryclose(log10[i], f) { + t.Errorf("Log10(%g) = %g, want %g", vc[i], f, log10[i]) + } + } + for i := 0; i < len(vcLog10SC); i++ { + if f := Log10(vcLog10SC[i]); !cAlike(log10SC[i], f) { + t.Errorf("Log10(%g) = %g, want %g", vcLog10SC[i], f, log10SC[i]) + } + } +} +func TestPolar(t *testing.T) { + for i := 0; i < len(vc); i++ { + if r, theta := Polar(vc[i]); !veryclose(polar[i].r, r) && !veryclose(polar[i].theta, theta) { + t.Errorf("Polar(%g) = %g, %g want %g, %g", vc[i], r, theta, polar[i].r, polar[i].theta) + } + } + for i := 0; i < len(vcPolarSC); i++ { + if r, theta := Polar(vcPolarSC[i]); !alike(polarSC[i].r, r) && !alike(polarSC[i].theta, theta) { + t.Errorf("Polar(%g) = %g, %g, want %g, %g", vcPolarSC[i], r, theta, polarSC[i].r, polarSC[i].theta) + } + } +} +func TestPow(t *testing.T) { + var a = complex(3.0, 3.0) + for i := 0; i < len(vc); i++ { + if f := Pow(a, vc[i]); !cSoclose(pow[i], f, 4e-15) { + t.Errorf("Pow(%g, %g) = %g, want %g", a, vc[i], f, pow[i]) + } + } + for i := 0; i < len(vcPowSC); i++ { + if f := Pow(vcPowSC[i][0], vcPowSC[i][0]); !cAlike(powSC[i], f) { + t.Errorf("Pow(%g, %g) = %g, want %g", vcPowSC[i][0], vcPowSC[i][0], f, powSC[i]) + } + } +} +func TestRect(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Rect(polar[i].r, polar[i].theta); !cVeryclose(vc[i], f) { + t.Errorf("Rect(%g, %g) = %g want %g", polar[i].r, polar[i].theta, f, vc[i]) + } + } + for i := 0; i < len(vcPolarSC); i++ { + if f := Rect(polarSC[i].r, polarSC[i].theta); !cAlike(vcPolarSC[i], f) { + t.Errorf("Rect(%g, %g) = %g, want %g", polarSC[i].r, polarSC[i].theta, f, vcPolarSC[i]) + } + } +} +func TestSin(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Sin(vc[i]); !cSoclose(sin[i], f, 2e-15) { + t.Errorf("Sin(%g) = %g, want %g", vc[i], f, sin[i]) + } + } + for i := 0; i < len(vcSinSC); i++ { + if f := Sin(vcSinSC[i]); !cAlike(sinSC[i], f) { + t.Errorf("Sin(%g) = %g, want %g", vcSinSC[i], f, sinSC[i]) + } + } +} +func TestSinh(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Sinh(vc[i]); !cSoclose(sinh[i], f, 2e-15) { + t.Errorf("Sinh(%g) = %g, want %g", vc[i], f, sinh[i]) + } + } + for i := 0; i < len(vcSinhSC); i++ { + if f := Sinh(vcSinhSC[i]); !cAlike(sinhSC[i], f) { + t.Errorf("Sinh(%g) = %g, want %g", vcSinhSC[i], f, sinhSC[i]) + } + } +} +func TestSqrt(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Sqrt(vc[i]); !cVeryclose(sqrt[i], f) { + t.Errorf("Sqrt(%g) = %g, want %g", vc[i], f, sqrt[i]) + } + } + for i := 0; i < len(vcSqrtSC); i++ { + if f := Sqrt(vcSqrtSC[i]); !cAlike(sqrtSC[i], f) { + t.Errorf("Sqrt(%g) = %g, want %g", vcSqrtSC[i], f, sqrtSC[i]) + } + } +} +func TestTan(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Tan(vc[i]); !cSoclose(tan[i], f, 3e-15) { + t.Errorf("Tan(%g) = %g, want %g", vc[i], f, tan[i]) + } + } + for i := 0; i < len(vcTanSC); i++ { + if f := Tan(vcTanSC[i]); !cAlike(tanSC[i], f) { + t.Errorf("Tan(%g) = %g, want %g", vcTanSC[i], f, tanSC[i]) + } + } +} +func TestTanh(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Tanh(vc[i]); !cSoclose(tanh[i], f, 2e-15) { + t.Errorf("Tanh(%g) = %g, want %g", vc[i], f, tanh[i]) + } + } + for i := 0; i < len(vcTanhSC); i++ { + if f := Tanh(vcTanhSC[i]); !cAlike(tanhSC[i], f) { + t.Errorf("Tanh(%g) = %g, want %g", vcTanhSC[i], f, tanhSC[i]) + } + } +} + +func BenchmarkAbs(b *testing.B) { + for i := 0; i < b.N; i++ { + Abs(complex(2.5, 3.5)) + } +} +func BenchmarkAcos(b *testing.B) { + for i := 0; i < b.N; i++ { + Acos(complex(2.5, 3.5)) + } +} +func BenchmarkAcosh(b *testing.B) { + for i := 0; i < b.N; i++ { + Acosh(complex(2.5, 3.5)) + } +} +func BenchmarkAsin(b *testing.B) { + for i := 0; i < b.N; i++ { + Asin(complex(2.5, 3.5)) + } +} +func BenchmarkAsinh(b *testing.B) { + for i := 0; i < b.N; i++ { + Asinh(complex(2.5, 3.5)) + } +} +func BenchmarkAtan(b *testing.B) { + for i := 0; i < b.N; i++ { + Atan(complex(2.5, 3.5)) + } +} +func BenchmarkAtanh(b *testing.B) { + for i := 0; i < b.N; i++ { + Atanh(complex(2.5, 3.5)) + } +} +func BenchmarkConj(b *testing.B) { + for i := 0; i < b.N; i++ { + Conj(complex(2.5, 3.5)) + } +} +func BenchmarkCos(b *testing.B) { + for i := 0; i < b.N; i++ { + Cos(complex(2.5, 3.5)) + } +} +func BenchmarkCosh(b *testing.B) { + for i := 0; i < b.N; i++ { + Cosh(complex(2.5, 3.5)) + } +} +func BenchmarkExp(b *testing.B) { + for i := 0; i < b.N; i++ { + Exp(complex(2.5, 3.5)) + } +} +func BenchmarkLog(b *testing.B) { + for i := 0; i < b.N; i++ { + Log(complex(2.5, 3.5)) + } +} +func BenchmarkLog10(b *testing.B) { + for i := 0; i < b.N; i++ { + Log10(complex(2.5, 3.5)) + } +} +func BenchmarkPhase(b *testing.B) { + for i := 0; i < b.N; i++ { + Phase(complex(2.5, 3.5)) + } +} +func BenchmarkPolar(b *testing.B) { + for i := 0; i < b.N; i++ { + Polar(complex(2.5, 3.5)) + } +} +func BenchmarkPow(b *testing.B) { + for i := 0; i < b.N; i++ { + Pow(complex(2.5, 3.5), complex(2.5, 3.5)) + } +} +func BenchmarkRect(b *testing.B) { + for i := 0; i < b.N; i++ { + Rect(2.5, 1.5) + } +} +func BenchmarkSin(b *testing.B) { + for i := 0; i < b.N; i++ { + Sin(complex(2.5, 3.5)) + } +} +func BenchmarkSinh(b *testing.B) { + for i := 0; i < b.N; i++ { + Sinh(complex(2.5, 3.5)) + } +} +func BenchmarkSqrt(b *testing.B) { + for i := 0; i < b.N; i++ { + Sqrt(complex(2.5, 3.5)) + } +} +func BenchmarkTan(b *testing.B) { + for i := 0; i < b.N; i++ { + Tan(complex(2.5, 3.5)) + } +} +func BenchmarkTanh(b *testing.B) { + for i := 0; i < b.N; i++ { + Tanh(complex(2.5, 3.5)) + } +} diff --git a/libgo/go/cmath/conj.go b/libgo/go/cmath/conj.go new file mode 100644 index 000000000..776b57da7 --- /dev/null +++ b/libgo/go/cmath/conj.go @@ -0,0 +1,8 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cmath + +// Conj returns the complex conjugate of x. +func Conj(x complex128) complex128 { return complex(real(x), -imag(x)) } diff --git a/libgo/go/cmath/exp.go b/libgo/go/cmath/exp.go new file mode 100644 index 000000000..64c1ef409 --- /dev/null +++ b/libgo/go/cmath/exp.go @@ -0,0 +1,55 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cmath + +import "math" + +// The original C code, the long comment, and the constants +// below are from http://netlib.sandia.gov/cephes/c9x-complex/clog.c. +// The go code is a simplified version of the original C. +// +// Cephes Math Library Release 2.8: June, 2000 +// Copyright 1984, 1987, 1989, 1992, 2000 by Stephen L. Moshier +// +// The readme file at http://netlib.sandia.gov/cephes/ says: +// Some software in this archive may be from the book _Methods and +// Programs for Mathematical Functions_ (Prentice-Hall or Simon & Schuster +// International, 1989) or from the Cephes Mathematical Library, a +// commercial product. In either event, it is copyrighted by the author. +// What you see here may be used freely but it comes with no support or +// guarantee. +// +// The two known misprints in the book are repaired here in the +// source listings for the gamma function and the incomplete beta +// integral. +// +// Stephen L. Moshier +// moshier@na-net.ornl.gov + +// Complex exponential function +// +// DESCRIPTION: +// +// Returns the complex exponential of the complex argument z. +// +// If +// z = x + iy, +// r = exp(x), +// then +// w = r cos y + i r sin y. +// +// ACCURACY: +// +// Relative error: +// arithmetic domain # trials peak rms +// DEC -10,+10 8700 3.7e-17 1.1e-17 +// IEEE -10,+10 30000 3.0e-16 8.7e-17 + +// Exp returns e**x, the base-e exponential of x. +func Exp(x complex128) complex128 { + r := math.Exp(real(x)) + s, c := math.Sincos(imag(x)) + return complex(r*c, r*s) +} diff --git a/libgo/go/cmath/isinf.go b/libgo/go/cmath/isinf.go new file mode 100644 index 000000000..f23d2dea7 --- /dev/null +++ b/libgo/go/cmath/isinf.go @@ -0,0 +1,21 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cmath + +import "math" + +// IsInf returns true if either real(x) or imag(x) is an infinity. +func IsInf(x complex128) bool { + if math.IsInf(real(x), 0) || math.IsInf(imag(x), 0) { + return true + } + return false +} + +// Inf returns a complex infinity, complex(+Inf, +Inf). +func Inf() complex128 { + inf := math.Inf(1) + return complex(inf, inf) +} diff --git a/libgo/go/cmath/isnan.go b/libgo/go/cmath/isnan.go new file mode 100644 index 000000000..2063bb835 --- /dev/null +++ b/libgo/go/cmath/isnan.go @@ -0,0 +1,25 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cmath + +import "math" + +// IsNaN returns true if either real(x) or imag(x) is NaN +// and neither is an infinity. +func IsNaN(x complex128) bool { + switch { + case math.IsInf(real(x), 0) || math.IsInf(imag(x), 0): + return false + case math.IsNaN(real(x)) || math.IsNaN(imag(x)): + return true + } + return false +} + +// NaN returns a complex ``not-a-number'' value. +func NaN() complex128 { + nan := math.NaN() + return complex(nan, nan) +} diff --git a/libgo/go/cmath/log.go b/libgo/go/cmath/log.go new file mode 100644 index 000000000..8e6964fee --- /dev/null +++ b/libgo/go/cmath/log.go @@ -0,0 +1,64 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cmath + +import "math" + +// The original C code, the long comment, and the constants +// below are from http://netlib.sandia.gov/cephes/c9x-complex/clog.c. +// The go code is a simplified version of the original C. +// +// Cephes Math Library Release 2.8: June, 2000 +// Copyright 1984, 1987, 1989, 1992, 2000 by Stephen L. Moshier +// +// The readme file at http://netlib.sandia.gov/cephes/ says: +// Some software in this archive may be from the book _Methods and +// Programs for Mathematical Functions_ (Prentice-Hall or Simon & Schuster +// International, 1989) or from the Cephes Mathematical Library, a +// commercial product. In either event, it is copyrighted by the author. +// What you see here may be used freely but it comes with no support or +// guarantee. +// +// The two known misprints in the book are repaired here in the +// source listings for the gamma function and the incomplete beta +// integral. +// +// Stephen L. Moshier +// moshier@na-net.ornl.gov + +// Complex natural logarithm +// +// DESCRIPTION: +// +// Returns complex logarithm to the base e (2.718...) of +// the complex argument z. +// +// If +// z = x + iy, r = sqrt( x**2 + y**2 ), +// then +// w = log(r) + i arctan(y/x). +// +// The arctangent ranges from -PI to +PI. +// +// ACCURACY: +// +// Relative error: +// arithmetic domain # trials peak rms +// DEC -10,+10 7000 8.5e-17 1.9e-17 +// IEEE -10,+10 30000 5.0e-15 1.1e-16 +// +// Larger relative error can be observed for z near 1 +i0. +// In IEEE arithmetic the peak absolute error is 5.2e-16, rms +// absolute error 1.0e-16. + +// Log returns the natural logarithm of x. +func Log(x complex128) complex128 { + return complex(math.Log(Abs(x)), Phase(x)) +} + +// Log10 returns the decimal logarithm of x. +func Log10(x complex128) complex128 { + return math.Log10E * Log(x) +} diff --git a/libgo/go/cmath/phase.go b/libgo/go/cmath/phase.go new file mode 100644 index 000000000..2d67aa34c --- /dev/null +++ b/libgo/go/cmath/phase.go @@ -0,0 +1,11 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cmath + +import "math" + +// Phase returns the phase (also called the argument) of x. +// The returned value is in the range [-Pi, Pi]. +func Phase(x complex128) float64 { return math.Atan2(imag(x), real(x)) } diff --git a/libgo/go/cmath/polar.go b/libgo/go/cmath/polar.go new file mode 100644 index 000000000..033676acc --- /dev/null +++ b/libgo/go/cmath/polar.go @@ -0,0 +1,12 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cmath + +// Polar returns the absolute value r and phase θ of x, +// such that x = r * e**θi. +// The phase is in the range [-Pi, Pi]. +func Polar(x complex128) (r, θ float64) { + return Abs(x), Phase(x) +} diff --git a/libgo/go/cmath/pow.go b/libgo/go/cmath/pow.go new file mode 100644 index 000000000..68e1207c6 --- /dev/null +++ b/libgo/go/cmath/pow.go @@ -0,0 +1,60 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cmath + +import "math" + +// The original C code, the long comment, and the constants +// below are from http://netlib.sandia.gov/cephes/c9x-complex/clog.c. +// The go code is a simplified version of the original C. +// +// Cephes Math Library Release 2.8: June, 2000 +// Copyright 1984, 1987, 1989, 1992, 2000 by Stephen L. Moshier +// +// The readme file at http://netlib.sandia.gov/cephes/ says: +// Some software in this archive may be from the book _Methods and +// Programs for Mathematical Functions_ (Prentice-Hall or Simon & Schuster +// International, 1989) or from the Cephes Mathematical Library, a +// commercial product. In either event, it is copyrighted by the author. +// What you see here may be used freely but it comes with no support or +// guarantee. +// +// The two known misprints in the book are repaired here in the +// source listings for the gamma function and the incomplete beta +// integral. +// +// Stephen L. Moshier +// moshier@na-net.ornl.gov + +// Complex power function +// +// DESCRIPTION: +// +// Raises complex A to the complex Zth power. +// Definition is per AMS55 # 4.2.8, +// analytically equivalent to cpow(a,z) = cexp(z clog(a)). +// +// ACCURACY: +// +// Relative error: +// arithmetic domain # trials peak rms +// IEEE -10,+10 30000 9.4e-15 1.5e-15 + +// Pow returns x**y, the base-x exponential of y. +func Pow(x, y complex128) complex128 { + modulus := Abs(x) + if modulus == 0 { + return complex(0, 0) + } + r := math.Pow(modulus, real(y)) + arg := Phase(x) + theta := real(y) * arg + if imag(y) != 0 { + r *= math.Exp(-imag(y) * arg) + theta += imag(y) * math.Log(modulus) + } + s, c := math.Sincos(theta) + return complex(r*c, r*s) +} diff --git a/libgo/go/cmath/rect.go b/libgo/go/cmath/rect.go new file mode 100644 index 000000000..b955f0bf7 --- /dev/null +++ b/libgo/go/cmath/rect.go @@ -0,0 +1,13 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cmath + +import "math" + +// Rect returns the complex number x with polar coordinates r, θ. +func Rect(r, θ float64) complex128 { + s, c := math.Sincos(θ) + return complex(r*c, r*s) +} diff --git a/libgo/go/cmath/sin.go b/libgo/go/cmath/sin.go new file mode 100644 index 000000000..8900ecdde --- /dev/null +++ b/libgo/go/cmath/sin.go @@ -0,0 +1,132 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cmath + +import "math" + +// The original C code, the long comment, and the constants +// below are from http://netlib.sandia.gov/cephes/c9x-complex/clog.c. +// The go code is a simplified version of the original C. +// +// Cephes Math Library Release 2.8: June, 2000 +// Copyright 1984, 1987, 1989, 1992, 2000 by Stephen L. Moshier +// +// The readme file at http://netlib.sandia.gov/cephes/ says: +// Some software in this archive may be from the book _Methods and +// Programs for Mathematical Functions_ (Prentice-Hall or Simon & Schuster +// International, 1989) or from the Cephes Mathematical Library, a +// commercial product. In either event, it is copyrighted by the author. +// What you see here may be used freely but it comes with no support or +// guarantee. +// +// The two known misprints in the book are repaired here in the +// source listings for the gamma function and the incomplete beta +// integral. +// +// Stephen L. Moshier +// moshier@na-net.ornl.gov + +// Complex circular sine +// +// DESCRIPTION: +// +// If +// z = x + iy, +// +// then +// +// w = sin x cosh y + i cos x sinh y. +// +// csin(z) = -i csinh(iz). +// +// ACCURACY: +// +// Relative error: +// arithmetic domain # trials peak rms +// DEC -10,+10 8400 5.3e-17 1.3e-17 +// IEEE -10,+10 30000 3.8e-16 1.0e-16 +// Also tested by csin(casin(z)) = z. + +// Sin returns the sine of x. +func Sin(x complex128) complex128 { + s, c := math.Sincos(real(x)) + sh, ch := sinhcosh(imag(x)) + return complex(s*ch, c*sh) +} + +// Complex hyperbolic sine +// +// DESCRIPTION: +// +// csinh z = (cexp(z) - cexp(-z))/2 +// = sinh x * cos y + i cosh x * sin y . +// +// ACCURACY: +// +// Relative error: +// arithmetic domain # trials peak rms +// IEEE -10,+10 30000 3.1e-16 8.2e-17 + +// Sinh returns the hyperbolic sine of x. +func Sinh(x complex128) complex128 { + s, c := math.Sincos(imag(x)) + sh, ch := sinhcosh(real(x)) + return complex(c*sh, s*ch) +} + +// Complex circular cosine +// +// DESCRIPTION: +// +// If +// z = x + iy, +// +// then +// +// w = cos x cosh y - i sin x sinh y. +// +// ACCURACY: +// +// Relative error: +// arithmetic domain # trials peak rms +// DEC -10,+10 8400 4.5e-17 1.3e-17 +// IEEE -10,+10 30000 3.8e-16 1.0e-16 + +// Cos returns the cosine of x. +func Cos(x complex128) complex128 { + s, c := math.Sincos(real(x)) + sh, ch := sinhcosh(imag(x)) + return complex(c*ch, -s*sh) +} + +// Complex hyperbolic cosine +// +// DESCRIPTION: +// +// ccosh(z) = cosh x cos y + i sinh x sin y . +// +// ACCURACY: +// +// Relative error: +// arithmetic domain # trials peak rms +// IEEE -10,+10 30000 2.9e-16 8.1e-17 + +// Cosh returns the hyperbolic cosine of x. +func Cosh(x complex128) complex128 { + s, c := math.Sincos(imag(x)) + sh, ch := sinhcosh(real(x)) + return complex(c*ch, s*sh) +} + +// calculate sinh and cosh +func sinhcosh(x float64) (sh, ch float64) { + if math.Fabs(x) <= 0.5 { + return math.Sinh(x), math.Cosh(x) + } + e := math.Exp(x) + ei := 0.5 / e + e *= 0.5 + return e - ei, e + ei +} diff --git a/libgo/go/cmath/sqrt.go b/libgo/go/cmath/sqrt.go new file mode 100644 index 000000000..e77a9b9df --- /dev/null +++ b/libgo/go/cmath/sqrt.go @@ -0,0 +1,103 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cmath + +import "math" + +// The original C code, the long comment, and the constants +// below are from http://netlib.sandia.gov/cephes/c9x-complex/clog.c. +// The go code is a simplified version of the original C. +// +// Cephes Math Library Release 2.8: June, 2000 +// Copyright 1984, 1987, 1989, 1992, 2000 by Stephen L. Moshier +// +// The readme file at http://netlib.sandia.gov/cephes/ says: +// Some software in this archive may be from the book _Methods and +// Programs for Mathematical Functions_ (Prentice-Hall or Simon & Schuster +// International, 1989) or from the Cephes Mathematical Library, a +// commercial product. In either event, it is copyrighted by the author. +// What you see here may be used freely but it comes with no support or +// guarantee. +// +// The two known misprints in the book are repaired here in the +// source listings for the gamma function and the incomplete beta +// integral. +// +// Stephen L. Moshier +// moshier@na-net.ornl.gov + +// Complex square root +// +// DESCRIPTION: +// +// If z = x + iy, r = |z|, then +// +// 1/2 +// Re w = [ (r + x)/2 ] , +// +// 1/2 +// Im w = [ (r - x)/2 ] . +// +// Cancellation error in r-x or r+x is avoided by using the +// identity 2 Re w Im w = y. +// +// Note that -w is also a square root of z. The root chosen +// is always in the right half plane and Im w has the same sign as y. +// +// ACCURACY: +// +// Relative error: +// arithmetic domain # trials peak rms +// DEC -10,+10 25000 3.2e-17 9.6e-18 +// IEEE -10,+10 1,000,000 2.9e-16 6.1e-17 + +// Sqrt returns the square root of x. +func Sqrt(x complex128) complex128 { + if imag(x) == 0 { + if real(x) == 0 { + return complex(0, 0) + } + if real(x) < 0 { + return complex(0, math.Sqrt(-real(x))) + } + return complex(math.Sqrt(real(x)), 0) + } + if real(x) == 0 { + if imag(x) < 0 { + r := math.Sqrt(-0.5 * imag(x)) + return complex(r, -r) + } + r := math.Sqrt(0.5 * imag(x)) + return complex(r, r) + } + a := real(x) + b := imag(x) + var scale float64 + // Rescale to avoid internal overflow or underflow. + if math.Fabs(a) > 4 || math.Fabs(b) > 4 { + a *= 0.25 + b *= 0.25 + scale = 2 + } else { + a *= 1.8014398509481984e16 // 2**54 + b *= 1.8014398509481984e16 + scale = 7.450580596923828125e-9 // 2**-27 + } + r := math.Hypot(a, b) + var t float64 + if a > 0 { + t = math.Sqrt(0.5*r + 0.5*a) + r = scale * math.Fabs((0.5*b)/t) + t *= scale + } else { + r = math.Sqrt(0.5*r - 0.5*a) + t = scale * math.Fabs((0.5*b)/r) + r *= scale + } + if b < 0 { + return complex(t, -r) + } + return complex(t, r) +} diff --git a/libgo/go/cmath/tan.go b/libgo/go/cmath/tan.go new file mode 100644 index 000000000..94b517521 --- /dev/null +++ b/libgo/go/cmath/tan.go @@ -0,0 +1,184 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cmath + +import "math" + +// The original C code, the long comment, and the constants +// below are from http://netlib.sandia.gov/cephes/c9x-complex/clog.c. +// The go code is a simplified version of the original C. +// +// Cephes Math Library Release 2.8: June, 2000 +// Copyright 1984, 1987, 1989, 1992, 2000 by Stephen L. Moshier +// +// The readme file at http://netlib.sandia.gov/cephes/ says: +// Some software in this archive may be from the book _Methods and +// Programs for Mathematical Functions_ (Prentice-Hall or Simon & Schuster +// International, 1989) or from the Cephes Mathematical Library, a +// commercial product. In either event, it is copyrighted by the author. +// What you see here may be used freely but it comes with no support or +// guarantee. +// +// The two known misprints in the book are repaired here in the +// source listings for the gamma function and the incomplete beta +// integral. +// +// Stephen L. Moshier +// moshier@na-net.ornl.gov + +// Complex circular tangent +// +// DESCRIPTION: +// +// If +// z = x + iy, +// +// then +// +// sin 2x + i sinh 2y +// w = --------------------. +// cos 2x + cosh 2y +// +// On the real axis the denominator is zero at odd multiples +// of PI/2. The denominator is evaluated by its Taylor +// series near these points. +// +// ctan(z) = -i ctanh(iz). +// +// ACCURACY: +// +// Relative error: +// arithmetic domain # trials peak rms +// DEC -10,+10 5200 7.1e-17 1.6e-17 +// IEEE -10,+10 30000 7.2e-16 1.2e-16 +// Also tested by ctan * ccot = 1 and catan(ctan(z)) = z. + +// Tan returns the tangent of x. +func Tan(x complex128) complex128 { + d := math.Cos(2*real(x)) + math.Cosh(2*imag(x)) + if math.Fabs(d) < 0.25 { + d = tanSeries(x) + } + if d == 0 { + return Inf() + } + return complex(math.Sin(2*real(x))/d, math.Sinh(2*imag(x))/d) +} + +// Complex hyperbolic tangent +// +// DESCRIPTION: +// +// tanh z = (sinh 2x + i sin 2y) / (cosh 2x + cos 2y) . +// +// ACCURACY: +// +// Relative error: +// arithmetic domain # trials peak rms +// IEEE -10,+10 30000 1.7e-14 2.4e-16 + +// Tanh returns the hyperbolic tangent of x. +func Tanh(x complex128) complex128 { + d := math.Cosh(2*real(x)) + math.Cos(2*imag(x)) + if d == 0 { + return Inf() + } + return complex(math.Sinh(2*real(x))/d, math.Sin(2*imag(x))/d) +} + +// Program to subtract nearest integer multiple of PI +func reducePi(x float64) float64 { + const ( + // extended precision value of PI: + DP1 = 3.14159265160560607910E0 // ?? 0x400921fb54000000 + DP2 = 1.98418714791870343106E-9 // ?? 0x3e210b4610000000 + DP3 = 1.14423774522196636802E-17 // ?? 0x3c6a62633145c06e + ) + t := x / math.Pi + if t >= 0 { + t += 0.5 + } else { + t -= 0.5 + } + t = float64(int64(t)) // int64(t) = the multiple + return ((x - t*DP1) - t*DP2) - t*DP3 +} + +// Taylor series expansion for cosh(2y) - cos(2x) +func tanSeries(z complex128) float64 { + const MACHEP = 1.0 / (1 << 53) + x := math.Fabs(2 * real(z)) + y := math.Fabs(2 * imag(z)) + x = reducePi(x) + x = x * x + y = y * y + x2 := 1.0 + y2 := 1.0 + f := 1.0 + rn := 0.0 + d := 0.0 + for { + rn += 1 + f *= rn + rn += 1 + f *= rn + x2 *= x + y2 *= y + t := y2 + x2 + t /= f + d += t + + rn += 1 + f *= rn + rn += 1 + f *= rn + x2 *= x + y2 *= y + t = y2 - x2 + t /= f + d += t + if math.Fabs(t/d) <= MACHEP { + break + } + } + return d +} + +// Complex circular cotangent +// +// DESCRIPTION: +// +// If +// z = x + iy, +// +// then +// +// sin 2x - i sinh 2y +// w = --------------------. +// cosh 2y - cos 2x +// +// On the real axis, the denominator has zeros at even +// multiples of PI/2. Near these points it is evaluated +// by a Taylor series. +// +// ACCURACY: +// +// Relative error: +// arithmetic domain # trials peak rms +// DEC -10,+10 3000 6.5e-17 1.6e-17 +// IEEE -10,+10 30000 9.2e-16 1.2e-16 +// Also tested by ctan * ccot = 1 + i0. + +// Cot returns the cotangent of x. +func Cot(x complex128) complex128 { + d := math.Cosh(2*imag(x)) - math.Cos(2*real(x)) + if math.Fabs(d) < 0.25 { + d = tanSeries(x) + } + if d == 0 { + return Inf() + } + return complex(math.Sin(2*real(x))/d, -math.Sinh(2*imag(x))/d) +} diff --git a/libgo/go/compress/flate/deflate.go b/libgo/go/compress/flate/deflate.go new file mode 100644 index 000000000..591b35c44 --- /dev/null +++ b/libgo/go/compress/flate/deflate.go @@ -0,0 +1,521 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package flate + +import ( + "io" + "math" + "os" +) + +const ( + NoCompression = 0 + BestSpeed = 1 + fastCompression = 3 + BestCompression = 9 + DefaultCompression = -1 + logMaxOffsetSize = 15 // Standard DEFLATE + wideLogMaxOffsetSize = 22 // Wide DEFLATE + minMatchLength = 3 // The smallest match that the compressor looks for + maxMatchLength = 258 // The longest match for the compressor + minOffsetSize = 1 // The shortest offset that makes any sence + + // The maximum number of tokens we put into a single flat block, just too + // stop things from getting too large. + maxFlateBlockTokens = 1 << 14 + maxStoreBlockSize = 65535 + hashBits = 15 + hashSize = 1 << hashBits + hashMask = (1 << hashBits) - 1 + hashShift = (hashBits + minMatchLength - 1) / minMatchLength +) + +type syncPipeReader struct { + *io.PipeReader + closeChan chan bool +} + +func (sr *syncPipeReader) CloseWithError(err os.Error) os.Error { + retErr := sr.PipeReader.CloseWithError(err) + sr.closeChan <- true // finish writer close + return retErr +} + +type syncPipeWriter struct { + *io.PipeWriter + closeChan chan bool +} + +type compressionLevel struct { + good, lazy, nice, chain, fastSkipHashing int +} + +var levels = []compressionLevel{ + {}, // 0 + // For levels 1-3 we don't bother trying with lazy matches + {3, 0, 8, 4, 4}, + {3, 0, 16, 8, 5}, + {3, 0, 32, 32, 6}, + // Levels 4-9 use increasingly more lazy matching + // and increasingly stringent conditions for "good enough". + {4, 4, 16, 16, math.MaxInt32}, + {8, 16, 32, 32, math.MaxInt32}, + {8, 16, 128, 128, math.MaxInt32}, + {8, 32, 128, 256, math.MaxInt32}, + {32, 128, 258, 1024, math.MaxInt32}, + {32, 258, 258, 4096, math.MaxInt32}, +} + +func (sw *syncPipeWriter) Close() os.Error { + err := sw.PipeWriter.Close() + <-sw.closeChan // wait for reader close + return err +} + +func syncPipe() (*syncPipeReader, *syncPipeWriter) { + r, w := io.Pipe() + sr := &syncPipeReader{r, make(chan bool, 1)} + sw := &syncPipeWriter{w, sr.closeChan} + return sr, sw +} + +type compressor struct { + level int + logWindowSize uint + w *huffmanBitWriter + r io.Reader + // (1 << logWindowSize) - 1. + windowMask int + + eof bool // has eof been reached on input? + sync bool // writer wants to flush + syncChan chan os.Error + + // hashHead[hashValue] contains the largest inputIndex with the specified hash value + hashHead []int + + // If hashHead[hashValue] is within the current window, then + // hashPrev[hashHead[hashValue] & windowMask] contains the previous index + // with the same hash value. + hashPrev []int + + // If we find a match of length >= niceMatch, then we don't bother searching + // any further. + niceMatch int + + // If we find a match of length >= goodMatch, we only do a half-hearted + // effort at doing lazy matching starting at the next character + goodMatch int + + // The maximum number of chains we look at when finding a match + maxChainLength int + + // The sliding window we use for matching + window []byte + + // The index just past the last valid character + windowEnd int + + // index in "window" at which current block starts + blockStart int +} + +func (d *compressor) flush() os.Error { + d.w.flush() + return d.w.err +} + +func (d *compressor) fillWindow(index int) (int, os.Error) { + if d.sync { + return index, nil + } + wSize := d.windowMask + 1 + if index >= wSize+wSize-(minMatchLength+maxMatchLength) { + // shift the window by wSize + copy(d.window, d.window[wSize:2*wSize]) + index -= wSize + d.windowEnd -= wSize + if d.blockStart >= wSize { + d.blockStart -= wSize + } else { + d.blockStart = math.MaxInt32 + } + for i, h := range d.hashHead { + d.hashHead[i] = max(h-wSize, -1) + } + for i, h := range d.hashPrev { + d.hashPrev[i] = max(h-wSize, -1) + } + } + count, err := d.r.Read(d.window[d.windowEnd:]) + d.windowEnd += count + if count == 0 && err == nil { + d.sync = true + } + if err == os.EOF { + d.eof = true + err = nil + } + return index, err +} + +func (d *compressor) writeBlock(tokens []token, index int, eof bool) os.Error { + if index > 0 || eof { + var window []byte + if d.blockStart <= index { + window = d.window[d.blockStart:index] + } + d.blockStart = index + d.w.writeBlock(tokens, eof, window) + return d.w.err + } + return nil +} + +// Try to find a match starting at index whose length is greater than prevSize. +// We only look at chainCount possibilities before giving up. +func (d *compressor) findMatch(pos int, prevHead int, prevLength int, lookahead int) (length, offset int, ok bool) { + win := d.window[0 : pos+min(maxMatchLength, lookahead)] + + // We quit when we get a match that's at least nice long + nice := min(d.niceMatch, len(win)-pos) + + // If we've got a match that's good enough, only look in 1/4 the chain. + tries := d.maxChainLength + length = prevLength + if length >= d.goodMatch { + tries >>= 2 + } + + w0 := win[pos] + w1 := win[pos+1] + wEnd := win[pos+length] + minIndex := pos - (d.windowMask + 1) + + for i := prevHead; tries > 0; tries-- { + if w0 == win[i] && w1 == win[i+1] && wEnd == win[i+length] { + // The hash function ensures that if win[i] and win[i+1] match, win[i+2] matches + + n := 3 + for pos+n < len(win) && win[i+n] == win[pos+n] { + n++ + } + if n > length && (n > 3 || pos-i <= 4096) { + length = n + offset = pos - i + ok = true + if n >= nice { + // The match is good enough that we don't try to find a better one. + break + } + wEnd = win[pos+n] + } + } + if i == minIndex { + // hashPrev[i & windowMask] has already been overwritten, so stop now. + break + } + if i = d.hashPrev[i&d.windowMask]; i < minIndex || i < 0 { + break + } + } + return +} + +func (d *compressor) writeStoredBlock(buf []byte) os.Error { + if d.w.writeStoredHeader(len(buf), false); d.w.err != nil { + return d.w.err + } + d.w.writeBytes(buf) + return d.w.err +} + +func (d *compressor) storedDeflate() os.Error { + buf := make([]byte, maxStoreBlockSize) + for { + n, err := d.r.Read(buf) + if n == 0 && err == nil { + d.sync = true + } + if n > 0 || d.sync { + if err := d.writeStoredBlock(buf[0:n]); err != nil { + return err + } + if d.sync { + d.syncChan <- nil + d.sync = false + } + } + if err != nil { + if err == os.EOF { + break + } + return err + } + } + return nil +} + +func (d *compressor) doDeflate() (err os.Error) { + // init + d.windowMask = 1< windowEnd { + panic("index > windowEnd") + } + lookahead := windowEnd - index + if lookahead < minMatchLength+maxMatchLength { + if index, err = d.fillWindow(index); err != nil { + return + } + windowEnd = d.windowEnd + if index > windowEnd { + panic("index > windowEnd") + } + maxInsertIndex = windowEnd - (minMatchLength - 1) + lookahead = windowEnd - index + if lookahead == 0 { + // Flush current output block if any. + if byteAvailable { + // There is still one pending token that needs to be flushed + tokens[ti] = literalToken(uint32(d.window[index-1]) & 0xFF) + ti++ + byteAvailable = false + } + if ti > 0 { + if err = d.writeBlock(tokens[0:ti], index, false); err != nil { + return + } + ti = 0 + } + if d.sync { + d.w.writeStoredHeader(0, false) + d.w.flush() + d.syncChan <- d.w.err + d.sync = false + } + + // If this was only a sync (not at EOF) keep going. + if !d.eof { + continue + } + break Loop + } + } + if index < maxInsertIndex { + // Update the hash + hash = (hash<= minIndex && + (isFastDeflate && lookahead > minMatchLength-1 || + !isFastDeflate && lookahead > prevLength && prevLength < lazyMatch) { + if newLength, newOffset, ok := d.findMatch(index, chainHead, minMatchLength-1, lookahead); ok { + length = newLength + offset = newOffset + } + } + if isFastDeflate && length >= minMatchLength || + !isFastDeflate && prevLength >= minMatchLength && length <= prevLength { + // There was a match at the previous step, and the current match is + // not better. Output the previous match. + if isFastDeflate { + tokens[ti] = matchToken(uint32(length-minMatchLength), uint32(offset-minOffsetSize)) + } else { + tokens[ti] = matchToken(uint32(prevLength-minMatchLength), uint32(prevOffset-minOffsetSize)) + } + ti++ + // Insert in the hash table all strings up to the end of the match. + // index and index-1 are already inserted. If there is not enough + // lookahead, the last two strings are not inserted into the hash + // table. + if length <= l.fastSkipHashing { + var newIndex int + if isFastDeflate { + newIndex = index + length + } else { + newIndex = prevLength - 1 + } + for index++; index < newIndex; index++ { + if index < maxInsertIndex { + hash = (hash< 0 || b.closed { + return + } + <-b.ready + } + panic("unreachable") +} + +func (b *syncBuffer) Write(p []byte) (n int, err os.Error) { + n, err = b.buf.Write(p) + _ = b.ready <- true + return +} + +func (b *syncBuffer) WriteMode() { + b.mu.Lock() +} + +func (b *syncBuffer) ReadMode() { + b.mu.Unlock() + _ = b.ready <- true +} + +func (b *syncBuffer) Close() os.Error { + b.closed = true + _ = b.ready <- true + return nil +} + +func testSync(t *testing.T, level int, input []byte, name string) { + if len(input) == 0 { + return + } + + t.Logf("--testSync %d, %d, %s", level, len(input), name) + buf := newSyncBuffer() + buf1 := new(bytes.Buffer) + buf.WriteMode() + w := NewWriter(io.MultiWriter(buf, buf1), level) + r := NewReader(buf) + + // Write half the input and read back. + for i := 0; i < 2; i++ { + var lo, hi int + if i == 0 { + lo, hi = 0, (len(input)+1)/2 + } else { + lo, hi = (len(input)+1)/2, len(input) + } + t.Logf("#%d: write %d-%d", i, lo, hi) + if _, err := w.Write(input[lo:hi]); err != nil { + t.Errorf("testSync: write: %v", err) + return + } + if i == 0 { + if err := w.Flush(); err != nil { + t.Errorf("testSync: flush: %v", err) + return + } + } else { + if err := w.Close(); err != nil { + t.Errorf("testSync: close: %v", err) + } + } + buf.ReadMode() + out := make([]byte, hi-lo+1) + m, err := io.ReadAtLeast(r, out, hi-lo) + t.Logf("#%d: read %d", i, m) + if m != hi-lo || err != nil { + t.Errorf("testSync/%d (%d, %d, %s): read %d: %d, %v (%d left)", i, level, len(input), name, hi-lo, m, err, buf.buf.Len()) + return + } + if !bytes.Equal(input[lo:hi], out[:hi-lo]) { + t.Errorf("testSync/%d: read wrong bytes: %x vs %x", i, input[lo:hi], out[:hi-lo]) + return + } + if i == 0 && buf.buf.Len() != 0 { + t.Errorf("testSync/%d (%d, %d, %s): extra data after %d", i, level, len(input), name, hi-lo) + } + buf.WriteMode() + } + buf.ReadMode() + out := make([]byte, 10) + if n, err := r.Read(out); n > 0 || err != os.EOF { + t.Errorf("testSync (%d, %d, %s): final Read: %d, %v (hex: %x)", level, len(input), name, n, err, out[0:n]) + } + if buf.buf.Len() != 0 { + t.Errorf("testSync (%d, %d, %s): extra data at end", level, len(input), name) + } + r.Close() + + // stream should work for ordinary reader too + r = NewReader(buf1) + out, err := ioutil.ReadAll(r) + if err != nil { + t.Errorf("testSync: read: %s", err) + return + } + r.Close() + if !bytes.Equal(input, out) { + t.Errorf("testSync: decompress(compress(data)) != data: level=%d input=%s", level, name) + } +} + + +func testToFromWithLevel(t *testing.T, level int, input []byte, name string) os.Error { + buffer := bytes.NewBuffer(nil) + w := NewWriter(buffer, level) + w.Write(input) + w.Close() + r := NewReader(buffer) + out, err := ioutil.ReadAll(r) + if err != nil { + t.Errorf("read: %s", err) + return err + } + r.Close() + if !bytes.Equal(input, out) { + t.Errorf("decompress(compress(data)) != data: level=%d input=%s", level, name) + } + + testSync(t, level, input, name) + return nil +} + +func testToFrom(t *testing.T, input []byte, name string) { + for i := 0; i < 10; i++ { + testToFromWithLevel(t, i, input, name) + } +} + +func TestDeflateInflate(t *testing.T) { + for i, h := range deflateInflateTests { + testToFrom(t, h.in, fmt.Sprintf("#%d", i)) + } +} + +func TestReverseBits(t *testing.T) { + for _, h := range reverseBitsTests { + if v := reverseBits(h.in, h.bitCount); v != h.out { + t.Errorf("reverseBits(%v,%v) = %v, want %v", + h.in, h.bitCount, v, h.out) + } + } +} + +func TestDeflateInflateString(t *testing.T) { + gold := bytes.NewBufferString(getEdata()).Bytes() + testToFromWithLevel(t, 1, gold, "2.718281828...") +} + +func getEdata() string { + return "2.718281828459045235360287471352662497757247093699959574966967627724076630353547" + + "59457138217852516642742746639193200305992181741359662904357290033429526059563073" + + "81323286279434907632338298807531952510190115738341879307021540891499348841675092" + + "44761460668082264800168477411853742345442437107539077744992069551702761838606261" + + "33138458300075204493382656029760673711320070932870912744374704723069697720931014" + + "16928368190255151086574637721112523897844250569536967707854499699679468644549059" + + "87931636889230098793127736178215424999229576351482208269895193668033182528869398" + + "49646510582093923982948879332036250944311730123819706841614039701983767932068328" + + "23764648042953118023287825098194558153017567173613320698112509961818815930416903" + + "51598888519345807273866738589422879228499892086805825749279610484198444363463244" + + "96848756023362482704197862320900216099023530436994184914631409343173814364054625" + + "31520961836908887070167683964243781405927145635490613031072085103837505101157477" + + "04171898610687396965521267154688957035035402123407849819334321068170121005627880" + + "23519303322474501585390473041995777709350366041699732972508868769664035557071622" + + "68447162560798826517871341951246652010305921236677194325278675398558944896970964" + + "09754591856956380236370162112047742722836489613422516445078182442352948636372141" + + "74023889344124796357437026375529444833799801612549227850925778256209262264832627" + + "79333865664816277251640191059004916449982893150566047258027786318641551956532442" + + "58698294695930801915298721172556347546396447910145904090586298496791287406870504" + + "89585867174798546677575732056812884592054133405392200011378630094556068816674001" + + "69842055804033637953764520304024322566135278369511778838638744396625322498506549" + + "95886234281899707733276171783928034946501434558897071942586398772754710962953741" + + "52111513683506275260232648472870392076431005958411661205452970302364725492966693" + + "81151373227536450988890313602057248176585118063036442812314965507047510254465011" + + "72721155519486685080036853228183152196003735625279449515828418829478761085263981" + + "39559900673764829224437528718462457803619298197139914756448826260390338144182326" + + "25150974827987779964373089970388867782271383605772978824125611907176639465070633" + + "04527954661855096666185664709711344474016070462621568071748187784437143698821855" + + "96709591025968620023537185887485696522000503117343920732113908032936344797273559" + + "55277349071783793421637012050054513263835440001863239914907054797780566978533580" + + "48966906295119432473099587655236812859041383241160722602998330535370876138939639" + + "17795745401613722361878936526053815584158718692553860616477983402543512843961294" + + "60352913325942794904337299085731580290958631382683291477116396337092400316894586" + + "36060645845925126994655724839186564209752685082307544254599376917041977780085362" + + "73094171016343490769642372229435236612557250881477922315197477806056967253801718" + + "07763603462459278778465850656050780844211529697521890874019660906651803516501792" + + "50461950136658543663271254963990854914420001457476081930221206602433009641270489" + + "43903971771951806990869986066365832322787093765022601492910115171776359446020232" + + "49300280401867723910288097866605651183260043688508817157238669842242201024950551" + + "88169480322100251542649463981287367765892768816359831247788652014117411091360116" + + "49950766290779436460058519419985601626479076153210387275571269925182756879893027" + + "61761146162549356495903798045838182323368612016243736569846703785853305275833337" + + "93990752166069238053369887956513728559388349989470741618155012539706464817194670" + + "83481972144888987906765037959036696724949925452790337296361626589760394985767413" + + "97359441023744329709355477982629614591442936451428617158587339746791897571211956" + + "18738578364475844842355558105002561149239151889309946342841393608038309166281881" + + "15037152849670597416256282360921680751501777253874025642534708790891372917228286" + + "11515915683725241630772254406337875931059826760944203261924285317018781772960235" + + "41306067213604600038966109364709514141718577701418060644363681546444005331608778" + + "31431744408119494229755993140118886833148328027065538330046932901157441475631399" + + "97221703804617092894579096271662260740718749975359212756084414737823303270330168" + + "23719364800217328573493594756433412994302485023573221459784328264142168487872167" + + "33670106150942434569844018733128101079451272237378861260581656680537143961278887" + + "32527373890392890506865324138062796025930387727697783792868409325365880733988457" + + "21874602100531148335132385004782716937621800490479559795929059165547050577751430" + + "81751126989851884087185640260353055837378324229241856256442550226721559802740126" + + "17971928047139600689163828665277009752767069777036439260224372841840883251848770" + + "47263844037953016690546593746161932384036389313136432713768884102681121989127522" + + "30562567562547017250863497653672886059667527408686274079128565769963137897530346" + + "60616669804218267724560530660773899624218340859882071864682623215080288286359746" + + "83965435885668550377313129658797581050121491620765676995065971534476347032085321" + + "56036748286083786568030730626576334697742956346437167093971930608769634953288468" + + "33613038829431040800296873869117066666146800015121143442256023874474325250769387" + + "07777519329994213727721125884360871583483562696166198057252661220679754062106208" + + "06498829184543953015299820925030054982570433905535701686531205264956148572492573" + + "86206917403695213533732531666345466588597286659451136441370331393672118569553952" + + "10845840724432383558606310680696492485123263269951460359603729725319836842336390" + + "46321367101161928217111502828016044880588023820319814930963695967358327420249882" + + "45684941273860566491352526706046234450549227581151709314921879592718001940968866" + + "98683703730220047531433818109270803001720593553052070070607223399946399057131158" + + "70996357773590271962850611465148375262095653467132900259943976631145459026858989" + + "79115837093419370441155121920117164880566945938131183843765620627846310490346293" + + "95002945834116482411496975832601180073169943739350696629571241027323913874175492" + + "30718624545432220395527352952402459038057445028922468862853365422138157221311632" + + "88112052146489805180092024719391710555390113943316681515828843687606961102505171" + + "00739276238555338627255353883096067164466237092264680967125406186950214317621166" + + "81400975952814939072226011126811531083873176173232352636058381731510345957365382" + + "23534992935822836851007810884634349983518404451704270189381994243410090575376257" + + "76757111809008816418331920196262341628816652137471732547772778348877436651882875" + + "21566857195063719365653903894493664217640031215278702223664636357555035655769488" + + "86549500270853923617105502131147413744106134445544192101336172996285694899193369" + + "18472947858072915608851039678195942983318648075608367955149663644896559294818785" + + "17840387733262470519450504198477420141839477312028158868457072905440575106012852" + + "58056594703046836344592652552137008068752009593453607316226118728173928074623094" + + "68536782310609792159936001994623799343421068781349734695924646975250624695861690" + + "91785739765951993929939955675427146549104568607020990126068187049841780791739240" + + "71945996323060254707901774527513186809982284730860766536866855516467702911336827" + + "56310722334672611370549079536583453863719623585631261838715677411873852772292259" + + "47433737856955384562468010139057278710165129666367644518724656537304024436841408" + + "14488732957847348490003019477888020460324660842875351848364959195082888323206522" + + "12810419044804724794929134228495197002260131043006241071797150279343326340799596" + + "05314460532304885289729176598760166678119379323724538572096075822771784833616135" + + "82612896226118129455927462767137794487586753657544861407611931125958512655759734" + + "57301533364263076798544338576171533346232527057200530398828949903425956623297578" + + "24887350292591668258944568946559926584547626945287805165017206747854178879822768" + + "06536650641910973434528878338621726156269582654478205672987756426325321594294418" + + "03994321700009054265076309558846589517170914760743713689331946909098190450129030" + + "70995662266203031826493657336984195557769637876249188528656866076005660256054457" + + "11337286840205574416030837052312242587223438854123179481388550075689381124935386" + + "31863528708379984569261998179452336408742959118074745341955142035172618420084550" + + "91708456823682008977394558426792142734775608796442792027083121501564063413416171" + + "66448069815483764491573900121217041547872591998943825364950514771379399147205219" + + "52907939613762110723849429061635760459623125350606853765142311534966568371511660" + + "42207963944666211632551577290709784731562782775987881364919512574833287937715714" + + "59091064841642678309949723674420175862269402159407924480541255360431317992696739" + + "15754241929660731239376354213923061787675395871143610408940996608947141834069836" + + "29936753626215452472984642137528910798843813060955526227208375186298370667872244" + + "30195793793786072107254277289071732854874374355781966511716618330881129120245204" + + "04868220007234403502544820283425418788465360259150644527165770004452109773558589" + + "76226554849416217149895323834216001140629507184904277892585527430352213968356790" + + "18076406042138307308774460170842688272261177180842664333651780002171903449234264" + + "26629226145600433738386833555534345300426481847398921562708609565062934040526494" + + "32442614456659212912256488935696550091543064261342526684725949143142393988454324" + + "86327461842846655985332312210466259890141712103446084271616619001257195870793217" + + "56969854401339762209674945418540711844643394699016269835160784892451405894094639" + + "52678073545797003070511636825194877011897640028276484141605872061841852971891540" + + "19688253289309149665345753571427318482016384644832499037886069008072709327673127" + + "58196656394114896171683298045513972950668760474091542042842999354102582911350224" + + "16907694316685742425225090269390348148564513030699251995904363840284292674125734" + + "22447765584177886171737265462085498294498946787350929581652632072258992368768457" + + "01782303809656788311228930580914057261086588484587310165815116753332767488701482" + + "91674197015125597825727074064318086014281490241467804723275976842696339357735429" + + "30186739439716388611764209004068663398856841681003872389214483176070116684503887" + + "21236436704331409115573328018297798873659091665961240202177855885487617616198937" + + "07943800566633648843650891448055710397652146960276625835990519870423001794655367" + + "9" +} diff --git a/libgo/go/compress/flate/flate_test.go b/libgo/go/compress/flate/flate_test.go new file mode 100644 index 000000000..bfd3b8381 --- /dev/null +++ b/libgo/go/compress/flate/flate_test.go @@ -0,0 +1,139 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This test tests some internals of the flate package. +// The tests in package compress/gzip serve as the +// end-to-end test of the decompressor. + +package flate + +import ( + "bytes" + "reflect" + "testing" +) + +// The Huffman code lengths used by the fixed-format Huffman blocks. +var fixedHuffmanBits = [...]int{ + // 0-143 length 8 + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + + // 144-255 length 9 + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + + // 256-279 length 7 + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, + + // 280-287 length 8 + 8, 8, 8, 8, 8, 8, 8, 8, +} + +type InitDecoderTest struct { + in []int + out huffmanDecoder + ok bool +} + +var initDecoderTests = []*InitDecoderTest{ + // Example from Connell 1973, + &InitDecoderTest{ + []int{3, 5, 2, 4, 3, 5, 5, 4, 4, 3, 4, 5}, + huffmanDecoder{ + 2, 5, + [maxCodeLen + 1]int{2: 0, 4, 13, 31}, + [maxCodeLen + 1]int{2: 0, 1, 6, 20}, + // Paper used different code assignment: + // 2, 9, 4, 0, 10, 8, 3, 7, 1, 5, 11, 6 + // Reordered here so that codes of same length + // are assigned to increasing numbers. + []int{2, 0, 4, 9, 3, 7, 8, 10, 1, 5, 6, 11}, + }, + true, + }, + + // Example from RFC 1951 section 3.2.2 + &InitDecoderTest{ + []int{2, 1, 3, 3}, + huffmanDecoder{ + 1, 3, + [maxCodeLen + 1]int{1: 0, 2, 7}, + [maxCodeLen + 1]int{1: 0, 1, 4}, + []int{1, 0, 2, 3}, + }, + true, + }, + + // Second example from RFC 1951 section 3.2.2 + &InitDecoderTest{ + []int{3, 3, 3, 3, 3, 2, 4, 4}, + huffmanDecoder{ + 2, 4, + [maxCodeLen + 1]int{2: 0, 6, 15}, + [maxCodeLen + 1]int{2: 0, 1, 8}, + []int{5, 0, 1, 2, 3, 4, 6, 7}, + }, + true, + }, + + // Static Huffman codes (RFC 1951 section 3.2.6) + &InitDecoderTest{ + fixedHuffmanBits[0:], + fixedHuffmanDecoder, + true, + }, + + // Illegal input. + &InitDecoderTest{ + []int{}, + huffmanDecoder{}, + false, + }, + + // Illegal input. + &InitDecoderTest{ + []int{0, 0, 0, 0, 0, 0, 0}, + huffmanDecoder{}, + false, + }, +} + +func TestInitDecoder(t *testing.T) { + for i, tt := range initDecoderTests { + var h huffmanDecoder + if h.init(tt.in) != tt.ok { + t.Errorf("test %d: init = %v", i, !tt.ok) + continue + } + if !reflect.DeepEqual(&h, &tt.out) { + t.Errorf("test %d:\nhave %v\nwant %v", i, h, tt.out) + } + } +} + +func TestUncompressedSource(t *testing.T) { + decoder := NewReader(bytes.NewBuffer([]byte{0x01, 0x01, 0x00, 0xfe, 0xff, 0x11})) + output := make([]byte, 1) + n, error := decoder.Read(output) + if n != 1 || error != nil { + t.Fatalf("decoder.Read() = %d, %v, want 1, nil", n, error) + } + if output[0] != 0x11 { + t.Errorf("output[0] = %x, want 0x11", output[0]) + } +} diff --git a/libgo/go/compress/flate/huffman_bit_writer.go b/libgo/go/compress/flate/huffman_bit_writer.go new file mode 100644 index 000000000..abff82dd6 --- /dev/null +++ b/libgo/go/compress/flate/huffman_bit_writer.go @@ -0,0 +1,506 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package flate + +import ( + "io" + "math" + "os" + "strconv" +) + +const ( + // The largest offset code. + offsetCodeCount = 30 + + // The largest offset code in the extensions. + extendedOffsetCodeCount = 42 + + // The special code used to mark the end of a block. + endBlockMarker = 256 + + // The first length code. + lengthCodesStart = 257 + + // The number of codegen codes. + codegenCodeCount = 19 + badCode = 255 +) + +// The number of extra bits needed by length code X - LENGTH_CODES_START. +var lengthExtraBits = []int8{ + /* 257 */ 0, 0, 0, + /* 260 */ 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, + /* 270 */ 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, + /* 280 */ 4, 5, 5, 5, 5, 0, +} + +// The length indicated by length code X - LENGTH_CODES_START. +var lengthBase = []uint32{ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, + 12, 14, 16, 20, 24, 28, 32, 40, 48, 56, + 64, 80, 96, 112, 128, 160, 192, 224, 255, +} + +// offset code word extra bits. +var offsetExtraBits = []int8{ + 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, + 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, + 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, + /* extended window */ + 14, 14, 15, 15, 16, 16, 17, 17, 18, 18, 19, 19, 20, 20, +} + +var offsetBase = []uint32{ + /* normal deflate */ + 0x000000, 0x000001, 0x000002, 0x000003, 0x000004, + 0x000006, 0x000008, 0x00000c, 0x000010, 0x000018, + 0x000020, 0x000030, 0x000040, 0x000060, 0x000080, + 0x0000c0, 0x000100, 0x000180, 0x000200, 0x000300, + 0x000400, 0x000600, 0x000800, 0x000c00, 0x001000, + 0x001800, 0x002000, 0x003000, 0x004000, 0x006000, + + /* extended window */ + 0x008000, 0x00c000, 0x010000, 0x018000, 0x020000, + 0x030000, 0x040000, 0x060000, 0x080000, 0x0c0000, + 0x100000, 0x180000, 0x200000, 0x300000, +} + +// The odd order in which the codegen code sizes are written. +var codegenOrder = []uint32{16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15} + +type huffmanBitWriter struct { + w io.Writer + // Data waiting to be written is bytes[0:nbytes] + // and then the low nbits of bits. + bits uint32 + nbits uint32 + bytes [64]byte + nbytes int + literalFreq []int32 + offsetFreq []int32 + codegen []uint8 + codegenFreq []int32 + literalEncoding *huffmanEncoder + offsetEncoding *huffmanEncoder + codegenEncoding *huffmanEncoder + err os.Error +} + +type WrongValueError struct { + name string + from int32 + to int32 + value int32 +} + +func newHuffmanBitWriter(w io.Writer) *huffmanBitWriter { + return &huffmanBitWriter{ + w: w, + literalFreq: make([]int32, maxLit), + offsetFreq: make([]int32, extendedOffsetCodeCount), + codegen: make([]uint8, maxLit+extendedOffsetCodeCount+1), + codegenFreq: make([]int32, codegenCodeCount), + literalEncoding: newHuffmanEncoder(maxLit), + offsetEncoding: newHuffmanEncoder(extendedOffsetCodeCount), + codegenEncoding: newHuffmanEncoder(codegenCodeCount), + } +} + +func (err WrongValueError) String() string { + return "huffmanBitWriter: " + err.name + " should belong to [" + strconv.Itoa64(int64(err.from)) + ";" + + strconv.Itoa64(int64(err.to)) + "] but actual value is " + strconv.Itoa64(int64(err.value)) +} + +func (w *huffmanBitWriter) flushBits() { + if w.err != nil { + w.nbits = 0 + return + } + bits := w.bits + w.bits >>= 16 + w.nbits -= 16 + n := w.nbytes + w.bytes[n] = byte(bits) + w.bytes[n+1] = byte(bits >> 8) + if n += 2; n >= len(w.bytes) { + _, w.err = w.w.Write(w.bytes[0:]) + n = 0 + } + w.nbytes = n +} + +func (w *huffmanBitWriter) flush() { + if w.err != nil { + w.nbits = 0 + return + } + n := w.nbytes + if w.nbits > 8 { + w.bytes[n] = byte(w.bits) + w.bits >>= 8 + w.nbits -= 8 + n++ + } + if w.nbits > 0 { + w.bytes[n] = byte(w.bits) + w.nbits = 0 + n++ + } + w.bits = 0 + _, w.err = w.w.Write(w.bytes[0:n]) + w.nbytes = 0 +} + +func (w *huffmanBitWriter) writeBits(b, nb int32) { + w.bits |= uint32(b) << w.nbits + if w.nbits += uint32(nb); w.nbits >= 16 { + w.flushBits() + } +} + +func (w *huffmanBitWriter) writeBytes(bytes []byte) { + if w.err != nil { + return + } + n := w.nbytes + if w.nbits == 8 { + w.bytes[n] = byte(w.bits) + w.nbits = 0 + n++ + } + if w.nbits != 0 { + w.err = InternalError("writeBytes with unfinished bits") + return + } + if n != 0 { + _, w.err = w.w.Write(w.bytes[0:n]) + if w.err != nil { + return + } + } + w.nbytes = 0 + _, w.err = w.w.Write(bytes) +} + +// RFC 1951 3.2.7 specifies a special run-length encoding for specifiying +// the literal and offset lengths arrays (which are concatenated into a single +// array). This method generates that run-length encoding. +// +// The result is written into the codegen array, and the frequencies +// of each code is written into the codegenFreq array. +// Codes 0-15 are single byte codes. Codes 16-18 are followed by additional +// information. Code badCode is an end marker +// +// numLiterals The number of literals in literalEncoding +// numOffsets The number of offsets in offsetEncoding +func (w *huffmanBitWriter) generateCodegen(numLiterals int, numOffsets int) { + fillInt32s(w.codegenFreq, 0) + // Note that we are using codegen both as a temporary variable for holding + // a copy of the frequencies, and as the place where we put the result. + // This is fine because the output is always shorter than the input used + // so far. + codegen := w.codegen // cache + // Copy the concatenated code sizes to codegen. Put a marker at the end. + copyUint8s(codegen[0:numLiterals], w.literalEncoding.codeBits) + copyUint8s(codegen[numLiterals:numLiterals+numOffsets], w.offsetEncoding.codeBits) + codegen[numLiterals+numOffsets] = badCode + + size := codegen[0] + count := 1 + outIndex := 0 + for inIndex := 1; size != badCode; inIndex++ { + // INVARIANT: We have seen "count" copies of size that have not yet + // had output generated for them. + nextSize := codegen[inIndex] + if nextSize == size { + count++ + continue + } + // We need to generate codegen indicating "count" of size. + if size != 0 { + codegen[outIndex] = size + outIndex++ + w.codegenFreq[size]++ + count-- + for count >= 3 { + n := min(count, 6) + codegen[outIndex] = 16 + outIndex++ + codegen[outIndex] = uint8(n - 3) + outIndex++ + w.codegenFreq[16]++ + count -= n + } + } else { + for count >= 11 { + n := min(count, 138) + codegen[outIndex] = 18 + outIndex++ + codegen[outIndex] = uint8(n - 11) + outIndex++ + w.codegenFreq[18]++ + count -= n + } + if count >= 3 { + // count >= 3 && count <= 10 + codegen[outIndex] = 17 + outIndex++ + codegen[outIndex] = uint8(count - 3) + outIndex++ + w.codegenFreq[17]++ + count = 0 + } + } + count-- + for ; count >= 0; count-- { + codegen[outIndex] = size + outIndex++ + w.codegenFreq[size]++ + } + // Set up invariant for next time through the loop. + size = nextSize + count = 1 + } + // Marker indicating the end of the codegen. + codegen[outIndex] = badCode +} + +func (w *huffmanBitWriter) writeCode(code *huffmanEncoder, literal uint32) { + if w.err != nil { + return + } + w.writeBits(int32(code.code[literal]), int32(code.codeBits[literal])) +} + +// Write the header of a dynamic Huffman block to the output stream. +// +// numLiterals The number of literals specified in codegen +// numOffsets The number of offsets specified in codegen +// numCodegens Tne number of codegens used in codegen +func (w *huffmanBitWriter) writeDynamicHeader(numLiterals int, numOffsets int, numCodegens int, isEof bool) { + if w.err != nil { + return + } + var firstBits int32 = 4 + if isEof { + firstBits = 5 + } + w.writeBits(firstBits, 3) + w.writeBits(int32(numLiterals-257), 5) + if numOffsets > offsetCodeCount { + // Extended version of decompressor + w.writeBits(int32(offsetCodeCount+((numOffsets-(1+offsetCodeCount))>>3)), 5) + w.writeBits(int32((numOffsets-(1+offsetCodeCount))&0x7), 3) + } else { + w.writeBits(int32(numOffsets-1), 5) + } + w.writeBits(int32(numCodegens-4), 4) + + for i := 0; i < numCodegens; i++ { + value := w.codegenEncoding.codeBits[codegenOrder[i]] + w.writeBits(int32(value), 3) + } + + i := 0 + for { + var codeWord int = int(w.codegen[i]) + i++ + if codeWord == badCode { + break + } + // The low byte contains the actual code to generate. + w.writeCode(w.codegenEncoding, uint32(codeWord)) + + switch codeWord { + case 16: + w.writeBits(int32(w.codegen[i]), 2) + i++ + break + case 17: + w.writeBits(int32(w.codegen[i]), 3) + i++ + break + case 18: + w.writeBits(int32(w.codegen[i]), 7) + i++ + break + } + } +} + +func (w *huffmanBitWriter) writeStoredHeader(length int, isEof bool) { + if w.err != nil { + return + } + var flag int32 + if isEof { + flag = 1 + } + w.writeBits(flag, 3) + w.flush() + w.writeBits(int32(length), 16) + w.writeBits(int32(^uint16(length)), 16) +} + +func (w *huffmanBitWriter) writeFixedHeader(isEof bool) { + if w.err != nil { + return + } + // Indicate that we are a fixed Huffman block + var value int32 = 2 + if isEof { + value = 3 + } + w.writeBits(value, 3) +} + +func (w *huffmanBitWriter) writeBlock(tokens []token, eof bool, input []byte) { + if w.err != nil { + return + } + fillInt32s(w.literalFreq, 0) + fillInt32s(w.offsetFreq, 0) + + n := len(tokens) + tokens = tokens[0 : n+1] + tokens[n] = endBlockMarker + + totalLength := -1 // Subtract 1 for endBlock. + for _, t := range tokens { + switch t.typ() { + case literalType: + w.literalFreq[t.literal()]++ + totalLength++ + break + case matchType: + length := t.length() + offset := t.offset() + totalLength += int(length + 3) + w.literalFreq[lengthCodesStart+lengthCode(length)]++ + w.offsetFreq[offsetCode(offset)]++ + break + } + } + w.literalEncoding.generate(w.literalFreq, 15) + w.offsetEncoding.generate(w.offsetFreq, 15) + + // get the number of literals + numLiterals := len(w.literalFreq) + for w.literalFreq[numLiterals-1] == 0 { + numLiterals-- + } + // get the number of offsets + numOffsets := len(w.offsetFreq) + for numOffsets > 1 && w.offsetFreq[numOffsets-1] == 0 { + numOffsets-- + } + storedBytes := 0 + if input != nil { + storedBytes = len(input) + } + var extraBits int64 + var storedSize int64 + if storedBytes <= maxStoreBlockSize && input != nil { + storedSize = int64((storedBytes + 5) * 8) + // We only bother calculating the costs of the extra bits required by + // the length of offset fields (which will be the same for both fixed + // and dynamic encoding), if we need to compare those two encodings + // against stored encoding. + for lengthCode := lengthCodesStart + 8; lengthCode < numLiterals; lengthCode++ { + // First eight length codes have extra size = 0. + extraBits += int64(w.literalFreq[lengthCode]) * int64(lengthExtraBits[lengthCode-lengthCodesStart]) + } + for offsetCode := 4; offsetCode < numOffsets; offsetCode++ { + // First four offset codes have extra size = 0. + extraBits += int64(w.offsetFreq[offsetCode]) * int64(offsetExtraBits[offsetCode]) + } + } else { + storedSize = math.MaxInt32 + } + + // Figure out which generates smaller code, fixed Huffman, dynamic + // Huffman, or just storing the data. + var fixedSize int64 = math.MaxInt64 + if numOffsets <= offsetCodeCount { + fixedSize = int64(3) + + fixedLiteralEncoding.bitLength(w.literalFreq) + + fixedOffsetEncoding.bitLength(w.offsetFreq) + + extraBits + } + // Generate codegen and codegenFrequencies, which indicates how to encode + // the literalEncoding and the offsetEncoding. + w.generateCodegen(numLiterals, numOffsets) + w.codegenEncoding.generate(w.codegenFreq, 7) + numCodegens := len(w.codegenFreq) + for numCodegens > 4 && w.codegenFreq[codegenOrder[numCodegens-1]] == 0 { + numCodegens-- + } + extensionSummand := 0 + if numOffsets > offsetCodeCount { + extensionSummand = 3 + } + dynamicHeader := int64(3+5+5+4+(3*numCodegens)) + + // Following line is an extension. + int64(extensionSummand) + + w.codegenEncoding.bitLength(w.codegenFreq) + + int64(extraBits) + + int64(w.codegenFreq[16]*2) + + int64(w.codegenFreq[17]*3) + + int64(w.codegenFreq[18]*7) + dynamicSize := dynamicHeader + + w.literalEncoding.bitLength(w.literalFreq) + + w.offsetEncoding.bitLength(w.offsetFreq) + + if storedSize < fixedSize && storedSize < dynamicSize { + w.writeStoredHeader(storedBytes, eof) + w.writeBytes(input[0:storedBytes]) + return + } + var literalEncoding *huffmanEncoder + var offsetEncoding *huffmanEncoder + + if fixedSize <= dynamicSize { + w.writeFixedHeader(eof) + literalEncoding = fixedLiteralEncoding + offsetEncoding = fixedOffsetEncoding + } else { + // Write the header. + w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof) + literalEncoding = w.literalEncoding + offsetEncoding = w.offsetEncoding + } + + // Write the tokens. + for _, t := range tokens { + switch t.typ() { + case literalType: + w.writeCode(literalEncoding, t.literal()) + break + case matchType: + // Write the length + length := t.length() + lengthCode := lengthCode(length) + w.writeCode(literalEncoding, lengthCode+lengthCodesStart) + extraLengthBits := int32(lengthExtraBits[lengthCode]) + if extraLengthBits > 0 { + extraLength := int32(length - lengthBase[lengthCode]) + w.writeBits(extraLength, extraLengthBits) + } + // Write the offset + offset := t.offset() + offsetCode := offsetCode(offset) + w.writeCode(offsetEncoding, offsetCode) + extraOffsetBits := int32(offsetExtraBits[offsetCode]) + if extraOffsetBits > 0 { + extraOffset := int32(offset - offsetBase[offsetCode]) + w.writeBits(extraOffset, extraOffsetBits) + } + break + default: + panic("unknown token type: " + string(t)) + } + } +} diff --git a/libgo/go/compress/flate/huffman_code.go b/libgo/go/compress/flate/huffman_code.go new file mode 100644 index 000000000..6be605f0a --- /dev/null +++ b/libgo/go/compress/flate/huffman_code.go @@ -0,0 +1,373 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package flate + +import ( + "math" + "sort" +) + +type huffmanEncoder struct { + codeBits []uint8 + code []uint16 +} + +type literalNode struct { + literal uint16 + freq int32 +} + +type chain struct { + // The sum of the leaves in this tree + freq int32 + + // The number of literals to the left of this item at this level + leafCount int32 + + // The right child of this chain in the previous level. + up *chain +} + +type levelInfo struct { + // Our level. for better printing + level int32 + + // The most recent chain generated for this level + lastChain *chain + + // The frequency of the next character to add to this level + nextCharFreq int32 + + // The frequency of the next pair (from level below) to add to this level. + // Only valid if the "needed" value of the next lower level is 0. + nextPairFreq int32 + + // The number of chains remaining to generate for this level before moving + // up to the next level + needed int32 + + // The levelInfo for level+1 + up *levelInfo + + // The levelInfo for level-1 + down *levelInfo +} + +func maxNode() literalNode { return literalNode{math.MaxUint16, math.MaxInt32} } + +func newHuffmanEncoder(size int) *huffmanEncoder { + return &huffmanEncoder{make([]uint8, size), make([]uint16, size)} +} + +// Generates a HuffmanCode corresponding to the fixed literal table +func generateFixedLiteralEncoding() *huffmanEncoder { + h := newHuffmanEncoder(maxLit) + codeBits := h.codeBits + code := h.code + var ch uint16 + for ch = 0; ch < maxLit; ch++ { + var bits uint16 + var size uint8 + switch { + case ch < 144: + // size 8, 000110000 .. 10111111 + bits = ch + 48 + size = 8 + break + case ch < 256: + // size 9, 110010000 .. 111111111 + bits = ch + 400 - 144 + size = 9 + break + case ch < 280: + // size 7, 0000000 .. 0010111 + bits = ch - 256 + size = 7 + break + default: + // size 8, 11000000 .. 11000111 + bits = ch + 192 - 280 + size = 8 + } + codeBits[ch] = size + code[ch] = reverseBits(bits, size) + } + return h +} + +func generateFixedOffsetEncoding() *huffmanEncoder { + h := newHuffmanEncoder(30) + codeBits := h.codeBits + code := h.code + for ch := uint16(0); ch < 30; ch++ { + codeBits[ch] = 5 + code[ch] = reverseBits(ch, 5) + } + return h +} + +var fixedLiteralEncoding *huffmanEncoder = generateFixedLiteralEncoding() +var fixedOffsetEncoding *huffmanEncoder = generateFixedOffsetEncoding() + +func (h *huffmanEncoder) bitLength(freq []int32) int64 { + var total int64 + for i, f := range freq { + if f != 0 { + total += int64(f) * int64(h.codeBits[i]) + } + } + return total +} + +// Generate elements in the chain using an iterative algorithm. +func (h *huffmanEncoder) generateChains(top *levelInfo, list []literalNode) { + n := len(list) + list = list[0 : n+1] + list[n] = maxNode() + + l := top + for { + if l.nextPairFreq == math.MaxInt32 && l.nextCharFreq == math.MaxInt32 { + // We've run out of both leafs and pairs. + // End all calculations for this level. + // To m sure we never come back to this level or any lower level, + // set nextPairFreq impossibly large. + l.lastChain = nil + l.needed = 0 + l = l.up + l.nextPairFreq = math.MaxInt32 + continue + } + + prevFreq := l.lastChain.freq + if l.nextCharFreq < l.nextPairFreq { + // The next item on this row is a leaf node. + n := l.lastChain.leafCount + 1 + l.lastChain = &chain{l.nextCharFreq, n, l.lastChain.up} + l.nextCharFreq = list[n].freq + } else { + // The next item on this row is a pair from the previous row. + // nextPairFreq isn't valid until we generate two + // more values in the level below + l.lastChain = &chain{l.nextPairFreq, l.lastChain.leafCount, l.down.lastChain} + l.down.needed = 2 + } + + if l.needed--; l.needed == 0 { + // We've done everything we need to do for this level. + // Continue calculating one level up. Fill in nextPairFreq + // of that level with the sum of the two nodes we've just calculated on + // this level. + up := l.up + if up == nil { + // All done! + return + } + up.nextPairFreq = prevFreq + l.lastChain.freq + l = up + } else { + // If we stole from below, move down temporarily to replenish it. + for l.down.needed > 0 { + l = l.down + } + } + } +} + +// Return the number of literals assigned to each bit size in the Huffman encoding +// +// This method is only called when list.length >= 3 +// The cases of 0, 1, and 2 literals are handled by special case code. +// +// list An array of the literals with non-zero frequencies +// and their associated frequencies. The array is in order of increasing +// frequency, and has as its last element a special element with frequency +// MaxInt32 +// maxBits The maximum number of bits that should be used to encode any literal. +// return An integer array in which array[i] indicates the number of literals +// that should be encoded in i bits. +func (h *huffmanEncoder) bitCounts(list []literalNode, maxBits int32) []int32 { + n := int32(len(list)) + list = list[0 : n+1] + list[n] = maxNode() + + // The tree can't have greater depth than n - 1, no matter what. This + // saves a little bit of work in some small cases + maxBits = minInt32(maxBits, n-1) + + // Create information about each of the levels. + // A bogus "Level 0" whose sole purpose is so that + // level1.prev.needed==0. This makes level1.nextPairFreq + // be a legitimate value that never gets chosen. + top := &levelInfo{needed: 0} + chain2 := &chain{list[1].freq, 2, new(chain)} + for level := int32(1); level <= maxBits; level++ { + // For every level, the first two items are the first two characters. + // We initialize the levels as if we had already figured this out. + top = &levelInfo{ + level: level, + lastChain: chain2, + nextCharFreq: list[2].freq, + nextPairFreq: list[0].freq + list[1].freq, + down: top, + } + top.down.up = top + if level == 1 { + top.nextPairFreq = math.MaxInt32 + } + } + + // We need a total of 2*n - 2 items at top level and have already generated 2. + top.needed = 2*n - 4 + + l := top + for { + if l.nextPairFreq == math.MaxInt32 && l.nextCharFreq == math.MaxInt32 { + // We've run out of both leafs and pairs. + // End all calculations for this level. + // To m sure we never come back to this level or any lower level, + // set nextPairFreq impossibly large. + l.lastChain = nil + l.needed = 0 + l = l.up + l.nextPairFreq = math.MaxInt32 + continue + } + + prevFreq := l.lastChain.freq + if l.nextCharFreq < l.nextPairFreq { + // The next item on this row is a leaf node. + n := l.lastChain.leafCount + 1 + l.lastChain = &chain{l.nextCharFreq, n, l.lastChain.up} + l.nextCharFreq = list[n].freq + } else { + // The next item on this row is a pair from the previous row. + // nextPairFreq isn't valid until we generate two + // more values in the level below + l.lastChain = &chain{l.nextPairFreq, l.lastChain.leafCount, l.down.lastChain} + l.down.needed = 2 + } + + if l.needed--; l.needed == 0 { + // We've done everything we need to do for this level. + // Continue calculating one level up. Fill in nextPairFreq + // of that level with the sum of the two nodes we've just calculated on + // this level. + up := l.up + if up == nil { + // All done! + break + } + up.nextPairFreq = prevFreq + l.lastChain.freq + l = up + } else { + // If we stole from below, move down temporarily to replenish it. + for l.down.needed > 0 { + l = l.down + } + } + } + + // Somethings is wrong if at the end, the top level is null or hasn't used + // all of the leaves. + if top.lastChain.leafCount != n { + panic("top.lastChain.leafCount != n") + } + + bitCount := make([]int32, maxBits+1) + bits := 1 + for chain := top.lastChain; chain.up != nil; chain = chain.up { + // chain.leafCount gives the number of literals requiring at least "bits" + // bits to encode. + bitCount[bits] = chain.leafCount - chain.up.leafCount + bits++ + } + return bitCount +} + +// Look at the leaves and assign them a bit count and an encoding as specified +// in RFC 1951 3.2.2 +func (h *huffmanEncoder) assignEncodingAndSize(bitCount []int32, list []literalNode) { + code := uint16(0) + for n, bits := range bitCount { + code <<= 1 + if n == 0 || bits == 0 { + continue + } + // The literals list[len(list)-bits] .. list[len(list)-bits] + // are encoded using "bits" bits, and get the values + // code, code + 1, .... The code values are + // assigned in literal order (not frequency order). + chunk := list[len(list)-int(bits):] + sortByLiteral(chunk) + for _, node := range chunk { + h.codeBits[node.literal] = uint8(n) + h.code[node.literal] = reverseBits(code, uint8(n)) + code++ + } + list = list[0 : len(list)-int(bits)] + } +} + +// Update this Huffman Code object to be the minimum code for the specified frequency count. +// +// freq An array of frequencies, in which frequency[i] gives the frequency of literal i. +// maxBits The maximum number of bits to use for any literal. +func (h *huffmanEncoder) generate(freq []int32, maxBits int32) { + list := make([]literalNode, len(freq)+1) + // Number of non-zero literals + count := 0 + // Set list to be the set of all non-zero literals and their frequencies + for i, f := range freq { + if f != 0 { + list[count] = literalNode{uint16(i), f} + count++ + } else { + h.codeBits[i] = 0 + } + } + // If freq[] is shorter than codeBits[], fill rest of codeBits[] with zeros + h.codeBits = h.codeBits[0:len(freq)] + list = list[0:count] + if count <= 2 { + // Handle the small cases here, because they are awkward for the general case code. With + // two or fewer literals, everything has bit length 1. + for i, node := range list { + // "list" is in order of increasing literal value. + h.codeBits[node.literal] = 1 + h.code[node.literal] = uint16(i) + } + return + } + sortByFreq(list) + + // Get the number of literals for each bit count + bitCount := h.bitCounts(list, maxBits) + // And do the assignment + h.assignEncodingAndSize(bitCount, list) +} + +type literalNodeSorter struct { + a []literalNode + less func(i, j int) bool +} + +func (s literalNodeSorter) Len() int { return len(s.a) } + +func (s literalNodeSorter) Less(i, j int) bool { + return s.less(i, j) +} + +func (s literalNodeSorter) Swap(i, j int) { s.a[i], s.a[j] = s.a[j], s.a[i] } + +func sortByFreq(a []literalNode) { + s := &literalNodeSorter{a, func(i, j int) bool { return a[i].freq < a[j].freq }} + sort.Sort(s) +} + +func sortByLiteral(a []literalNode) { + s := &literalNodeSorter{a, func(i, j int) bool { return a[i].literal < a[j].literal }} + sort.Sort(s) +} diff --git a/libgo/go/compress/flate/inflate.go b/libgo/go/compress/flate/inflate.go new file mode 100644 index 000000000..7dc8cf93b --- /dev/null +++ b/libgo/go/compress/flate/inflate.go @@ -0,0 +1,620 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The flate package implements the DEFLATE compressed data +// format, described in RFC 1951. The gzip and zlib packages +// implement access to DEFLATE-based file formats. +package flate + +import ( + "bufio" + "io" + "os" + "strconv" +) + +const ( + maxCodeLen = 16 // max length of Huffman code + maxHist = 32768 // max history required + maxLit = 286 + maxDist = 32 + numCodes = 19 // number of codes in Huffman meta-code +) + +// A CorruptInputError reports the presence of corrupt input at a given offset. +type CorruptInputError int64 + +func (e CorruptInputError) String() string { + return "flate: corrupt input before offset " + strconv.Itoa64(int64(e)) +} + +// An InternalError reports an error in the flate code itself. +type InternalError string + +func (e InternalError) String() string { return "flate: internal error: " + string(e) } + +// A ReadError reports an error encountered while reading input. +type ReadError struct { + Offset int64 // byte offset where error occurred + Error os.Error // error returned by underlying Read +} + +func (e *ReadError) String() string { + return "flate: read error at offset " + strconv.Itoa64(e.Offset) + ": " + e.Error.String() +} + +// A WriteError reports an error encountered while writing output. +type WriteError struct { + Offset int64 // byte offset where error occurred + Error os.Error // error returned by underlying Write +} + +func (e *WriteError) String() string { + return "flate: write error at offset " + strconv.Itoa64(e.Offset) + ": " + e.Error.String() +} + +// Huffman decoder is based on +// J. Brian Connell, ``A Huffman-Shannon-Fano Code,'' +// Proceedings of the IEEE, 61(7) (July 1973), pp 1046-1047. +type huffmanDecoder struct { + // min, max code length + min, max int + + // limit[i] = largest code word of length i + // Given code v of length n, + // need more bits if v > limit[n]. + limit [maxCodeLen + 1]int + + // base[i] = smallest code word of length i - seq number + base [maxCodeLen + 1]int + + // codes[seq number] = output code. + // Given code v of length n, value is + // codes[v - base[n]]. + codes []int +} + +// Initialize Huffman decoding tables from array of code lengths. +func (h *huffmanDecoder) init(bits []int) bool { + // TODO(rsc): Return false sometimes. + + // Count number of codes of each length, + // compute min and max length. + var count [maxCodeLen + 1]int + var min, max int + for _, n := range bits { + if n == 0 { + continue + } + if min == 0 || n < min { + min = n + } + if n > max { + max = n + } + count[n]++ + } + if max == 0 { + return false + } + + h.min = min + h.max = max + + // For each code range, compute + // nextcode (first code of that length), + // limit (last code of that length), and + // base (offset from first code to sequence number). + code := 0 + seq := 0 + var nextcode [maxCodeLen]int + for i := min; i <= max; i++ { + n := count[i] + nextcode[i] = code + h.base[i] = code - seq + code += n + seq += n + h.limit[i] = code - 1 + code <<= 1 + } + + // Make array mapping sequence numbers to codes. + if len(h.codes) < len(bits) { + h.codes = make([]int, len(bits)) + } + for i, n := range bits { + if n == 0 { + continue + } + code := nextcode[n] + nextcode[n]++ + seq := code - h.base[n] + h.codes[seq] = i + } + return true +} + +// Hard-coded Huffman tables for DEFLATE algorithm. +// See RFC 1951, section 3.2.6. +var fixedHuffmanDecoder = huffmanDecoder{ + 7, 9, + [maxCodeLen + 1]int{7: 23, 199, 511}, + [maxCodeLen + 1]int{7: 0, 24, 224}, + []int{ + // length 7: 256-279 + 256, 257, 258, 259, 260, 261, 262, + 263, 264, 265, 266, 267, 268, 269, + 270, 271, 272, 273, 274, 275, 276, + 277, 278, 279, + + // length 8: 0-143 + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, + 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, + 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, + 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, + 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, + 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, + 92, 93, 94, 95, 96, 97, 98, 99, 100, + 101, 102, 103, 104, 105, 106, 107, 108, + 109, 110, 111, 112, 113, 114, 115, 116, + 117, 118, 119, 120, 121, 122, 123, 124, + 125, 126, 127, 128, 129, 130, 131, 132, + 133, 134, 135, 136, 137, 138, 139, 140, + 141, 142, 143, + + // length 8: 280-287 + 280, 281, 282, 283, 284, 285, 286, 287, + + // length 9: 144-255 + 144, 145, 146, 147, 148, 149, 150, 151, + 152, 153, 154, 155, 156, 157, 158, 159, + 160, 161, 162, 163, 164, 165, 166, 167, + 168, 169, 170, 171, 172, 173, 174, 175, + 176, 177, 178, 179, 180, 181, 182, 183, + 184, 185, 186, 187, 188, 189, 190, 191, + 192, 193, 194, 195, 196, 197, 198, 199, + 200, 201, 202, 203, 204, 205, 206, 207, + 208, 209, 210, 211, 212, 213, 214, 215, + 216, 217, 218, 219, 220, 221, 222, 223, + 224, 225, 226, 227, 228, 229, 230, 231, + 232, 233, 234, 235, 236, 237, 238, 239, + 240, 241, 242, 243, 244, 245, 246, 247, + 248, 249, 250, 251, 252, 253, 254, 255, + }, +} + +// The actual read interface needed by NewReader. +// If the passed in io.Reader does not also have ReadByte, +// the NewReader will introduce its own buffering. +type Reader interface { + io.Reader + ReadByte() (c byte, err os.Error) +} + +// Decompress state. +type decompressor struct { + // Input/output sources. + r Reader + w io.Writer + roffset int64 + woffset int64 + + // Input bits, in top of b. + b uint32 + nb uint + + // Huffman decoders for literal/length, distance. + h1, h2 huffmanDecoder + + // Length arrays used to define Huffman codes. + bits [maxLit + maxDist]int + codebits [numCodes]int + + // Output history, buffer. + hist [maxHist]byte + hp int // current output position in buffer + hw int // have written hist[0:hw] already + hfull bool // buffer has filled at least once + + // Temporary buffer (avoids repeated allocation). + buf [4]byte +} + +func (f *decompressor) inflate() (err os.Error) { + final := false + for err == nil && !final { + for f.nb < 1+2 { + if err = f.moreBits(); err != nil { + return + } + } + final = f.b&1 == 1 + f.b >>= 1 + typ := f.b & 3 + f.b >>= 2 + f.nb -= 1 + 2 + switch typ { + case 0: + err = f.dataBlock() + case 1: + // compressed, fixed Huffman tables + err = f.decodeBlock(&fixedHuffmanDecoder, nil) + case 2: + // compressed, dynamic Huffman tables + if err = f.readHuffman(); err == nil { + err = f.decodeBlock(&f.h1, &f.h2) + } + default: + // 3 is reserved. + err = CorruptInputError(f.roffset) + } + } + return +} + +// RFC 1951 section 3.2.7. +// Compression with dynamic Huffman codes + +var codeOrder = [...]int{16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15} + +func (f *decompressor) readHuffman() os.Error { + // HLIT[5], HDIST[5], HCLEN[4]. + for f.nb < 5+5+4 { + if err := f.moreBits(); err != nil { + return err + } + } + nlit := int(f.b&0x1F) + 257 + f.b >>= 5 + ndist := int(f.b&0x1F) + 1 + f.b >>= 5 + nclen := int(f.b&0xF) + 4 + f.b >>= 4 + f.nb -= 5 + 5 + 4 + + // (HCLEN+4)*3 bits: code lengths in the magic codeOrder order. + for i := 0; i < nclen; i++ { + for f.nb < 3 { + if err := f.moreBits(); err != nil { + return err + } + } + f.codebits[codeOrder[i]] = int(f.b & 0x7) + f.b >>= 3 + f.nb -= 3 + } + for i := nclen; i < len(codeOrder); i++ { + f.codebits[codeOrder[i]] = 0 + } + if !f.h1.init(f.codebits[0:]) { + return CorruptInputError(f.roffset) + } + + // HLIT + 257 code lengths, HDIST + 1 code lengths, + // using the code length Huffman code. + for i, n := 0, nlit+ndist; i < n; { + x, err := f.huffSym(&f.h1) + if err != nil { + return err + } + if x < 16 { + // Actual length. + f.bits[i] = x + i++ + continue + } + // Repeat previous length or zero. + var rep int + var nb uint + var b int + switch x { + default: + return InternalError("unexpected length code") + case 16: + rep = 3 + nb = 2 + if i == 0 { + return CorruptInputError(f.roffset) + } + b = f.bits[i-1] + case 17: + rep = 3 + nb = 3 + b = 0 + case 18: + rep = 11 + nb = 7 + b = 0 + } + for f.nb < nb { + if err := f.moreBits(); err != nil { + return err + } + } + rep += int(f.b & uint32(1<>= nb + f.nb -= nb + if i+rep > n { + return CorruptInputError(f.roffset) + } + for j := 0; j < rep; j++ { + f.bits[i] = b + i++ + } + } + + if !f.h1.init(f.bits[0:nlit]) || !f.h2.init(f.bits[nlit:nlit+ndist]) { + return CorruptInputError(f.roffset) + } + + return nil +} + +// Decode a single Huffman block from f. +// hl and hd are the Huffman states for the lit/length values +// and the distance values, respectively. If hd == nil, using the +// fixed distance encoding associated with fixed Huffman blocks. +func (f *decompressor) decodeBlock(hl, hd *huffmanDecoder) os.Error { + for { + v, err := f.huffSym(hl) + if err != nil { + return err + } + var n uint // number of bits extra + var length int + switch { + case v < 256: + f.hist[f.hp] = byte(v) + f.hp++ + if f.hp == len(f.hist) { + if err = f.flush(); err != nil { + return err + } + } + continue + case v == 256: + return nil + // otherwise, reference to older data + case v < 265: + length = v - (257 - 3) + n = 0 + case v < 269: + length = v*2 - (265*2 - 11) + n = 1 + case v < 273: + length = v*4 - (269*4 - 19) + n = 2 + case v < 277: + length = v*8 - (273*8 - 35) + n = 3 + case v < 281: + length = v*16 - (277*16 - 67) + n = 4 + case v < 285: + length = v*32 - (281*32 - 131) + n = 5 + default: + length = 258 + n = 0 + } + if n > 0 { + for f.nb < n { + if err = f.moreBits(); err != nil { + return err + } + } + length += int(f.b & uint32(1<>= n + f.nb -= n + } + + var dist int + if hd == nil { + for f.nb < 5 { + if err = f.moreBits(); err != nil { + return err + } + } + dist = int(reverseByte[(f.b&0x1F)<<3]) + f.b >>= 5 + f.nb -= 5 + } else { + if dist, err = f.huffSym(hd); err != nil { + return err + } + } + + switch { + case dist < 4: + dist++ + case dist >= 30: + return CorruptInputError(f.roffset) + default: + nb := uint(dist-2) >> 1 + // have 1 bit in bottom of dist, need nb more. + extra := (dist & 1) << nb + for f.nb < nb { + if err = f.moreBits(); err != nil { + return err + } + } + extra |= int(f.b & uint32(1<>= nb + f.nb -= nb + dist = 1<<(nb+1) + 1 + extra + } + + // Copy history[-dist:-dist+length] into output. + if dist > len(f.hist) { + return InternalError("bad history distance") + } + + // No check on length; encoding can be prescient. + if !f.hfull && dist > f.hp { + return CorruptInputError(f.roffset) + } + + p := f.hp - dist + if p < 0 { + p += len(f.hist) + } + for i := 0; i < length; i++ { + f.hist[f.hp] = f.hist[p] + f.hp++ + p++ + if f.hp == len(f.hist) { + if err = f.flush(); err != nil { + return err + } + } + if p == len(f.hist) { + p = 0 + } + } + } + panic("unreached") +} + +// Copy a single uncompressed data block from input to output. +func (f *decompressor) dataBlock() os.Error { + // Uncompressed. + // Discard current half-byte. + f.nb = 0 + f.b = 0 + + // Length then ones-complement of length. + nr, err := io.ReadFull(f.r, f.buf[0:4]) + f.roffset += int64(nr) + if err != nil { + return &ReadError{f.roffset, err} + } + n := int(f.buf[0]) | int(f.buf[1])<<8 + nn := int(f.buf[2]) | int(f.buf[3])<<8 + if uint16(nn) != uint16(^n) { + return CorruptInputError(f.roffset) + } + + if n == 0 { + // 0-length block means sync + return f.flush() + } + + // Read len bytes into history, + // writing as history fills. + for n > 0 { + m := len(f.hist) - f.hp + if m > n { + m = n + } + m, err := io.ReadFull(f.r, f.hist[f.hp:f.hp+m]) + f.roffset += int64(m) + if err != nil { + return &ReadError{f.roffset, err} + } + n -= m + f.hp += m + if f.hp == len(f.hist) { + if err = f.flush(); err != nil { + return err + } + } + } + return nil +} + +func (f *decompressor) moreBits() os.Error { + c, err := f.r.ReadByte() + if err != nil { + if err == os.EOF { + err = io.ErrUnexpectedEOF + } + return err + } + f.roffset++ + f.b |= uint32(c) << f.nb + f.nb += 8 + return nil +} + +// Read the next Huffman-encoded symbol from f according to h. +func (f *decompressor) huffSym(h *huffmanDecoder) (int, os.Error) { + for n := uint(h.min); n <= uint(h.max); n++ { + lim := h.limit[n] + if lim == -1 { + continue + } + for f.nb < n { + if err := f.moreBits(); err != nil { + return 0, err + } + } + v := int(f.b & uint32(1<>8]) | int(reverseByte[v&0xFF])<<8 // reverse bits + if v <= lim { + f.b >>= n + f.nb -= n + return h.codes[v-h.base[n]], nil + } + } + return 0, CorruptInputError(f.roffset) +} + +// Flush any buffered output to the underlying writer. +func (f *decompressor) flush() os.Error { + if f.hw == f.hp { + return nil + } + n, err := f.w.Write(f.hist[f.hw:f.hp]) + if n != f.hp-f.hw && err == nil { + err = io.ErrShortWrite + } + if err != nil { + return &WriteError{f.woffset, err} + } + f.woffset += int64(f.hp - f.hw) + f.hw = f.hp + if f.hp == len(f.hist) { + f.hp = 0 + f.hw = 0 + f.hfull = true + } + return nil +} + +func makeReader(r io.Reader) Reader { + if rr, ok := r.(Reader); ok { + return rr + } + return bufio.NewReader(r) +} + +// decompress reads DEFLATE-compressed data from r and writes +// the uncompressed data to w. +func (f *decompressor) decompress(r io.Reader, w io.Writer) os.Error { + f.r = makeReader(r) + f.w = w + f.woffset = 0 + if err := f.inflate(); err != nil { + return err + } + if err := f.flush(); err != nil { + return err + } + return nil +} + +// NewReader returns a new ReadCloser that can be used +// to read the uncompressed version of r. It is the caller's +// responsibility to call Close on the ReadCloser when +// finished reading. +func NewReader(r io.Reader) io.ReadCloser { + var f decompressor + pr, pw := io.Pipe() + go func() { pw.CloseWithError(f.decompress(r, pw)) }() + return pr +} diff --git a/libgo/go/compress/flate/reverse_bits.go b/libgo/go/compress/flate/reverse_bits.go new file mode 100644 index 000000000..c1a02720d --- /dev/null +++ b/libgo/go/compress/flate/reverse_bits.go @@ -0,0 +1,48 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package flate + +var reverseByte = [256]byte{ + 0x00, 0x80, 0x40, 0xc0, 0x20, 0xa0, 0x60, 0xe0, + 0x10, 0x90, 0x50, 0xd0, 0x30, 0xb0, 0x70, 0xf0, + 0x08, 0x88, 0x48, 0xc8, 0x28, 0xa8, 0x68, 0xe8, + 0x18, 0x98, 0x58, 0xd8, 0x38, 0xb8, 0x78, 0xf8, + 0x04, 0x84, 0x44, 0xc4, 0x24, 0xa4, 0x64, 0xe4, + 0x14, 0x94, 0x54, 0xd4, 0x34, 0xb4, 0x74, 0xf4, + 0x0c, 0x8c, 0x4c, 0xcc, 0x2c, 0xac, 0x6c, 0xec, + 0x1c, 0x9c, 0x5c, 0xdc, 0x3c, 0xbc, 0x7c, 0xfc, + 0x02, 0x82, 0x42, 0xc2, 0x22, 0xa2, 0x62, 0xe2, + 0x12, 0x92, 0x52, 0xd2, 0x32, 0xb2, 0x72, 0xf2, + 0x0a, 0x8a, 0x4a, 0xca, 0x2a, 0xaa, 0x6a, 0xea, + 0x1a, 0x9a, 0x5a, 0xda, 0x3a, 0xba, 0x7a, 0xfa, + 0x06, 0x86, 0x46, 0xc6, 0x26, 0xa6, 0x66, 0xe6, + 0x16, 0x96, 0x56, 0xd6, 0x36, 0xb6, 0x76, 0xf6, + 0x0e, 0x8e, 0x4e, 0xce, 0x2e, 0xae, 0x6e, 0xee, + 0x1e, 0x9e, 0x5e, 0xde, 0x3e, 0xbe, 0x7e, 0xfe, + 0x01, 0x81, 0x41, 0xc1, 0x21, 0xa1, 0x61, 0xe1, + 0x11, 0x91, 0x51, 0xd1, 0x31, 0xb1, 0x71, 0xf1, + 0x09, 0x89, 0x49, 0xc9, 0x29, 0xa9, 0x69, 0xe9, + 0x19, 0x99, 0x59, 0xd9, 0x39, 0xb9, 0x79, 0xf9, + 0x05, 0x85, 0x45, 0xc5, 0x25, 0xa5, 0x65, 0xe5, + 0x15, 0x95, 0x55, 0xd5, 0x35, 0xb5, 0x75, 0xf5, + 0x0d, 0x8d, 0x4d, 0xcd, 0x2d, 0xad, 0x6d, 0xed, + 0x1d, 0x9d, 0x5d, 0xdd, 0x3d, 0xbd, 0x7d, 0xfd, + 0x03, 0x83, 0x43, 0xc3, 0x23, 0xa3, 0x63, 0xe3, + 0x13, 0x93, 0x53, 0xd3, 0x33, 0xb3, 0x73, 0xf3, + 0x0b, 0x8b, 0x4b, 0xcb, 0x2b, 0xab, 0x6b, 0xeb, + 0x1b, 0x9b, 0x5b, 0xdb, 0x3b, 0xbb, 0x7b, 0xfb, + 0x07, 0x87, 0x47, 0xc7, 0x27, 0xa7, 0x67, 0xe7, + 0x17, 0x97, 0x57, 0xd7, 0x37, 0xb7, 0x77, 0xf7, + 0x0f, 0x8f, 0x4f, 0xcf, 0x2f, 0xaf, 0x6f, 0xef, + 0x1f, 0x9f, 0x5f, 0xdf, 0x3f, 0xbf, 0x7f, 0xff, +} + +func reverseUint16(v uint16) uint16 { + return uint16(reverseByte[v>>8]) | uint16(reverseByte[v&0xFF])<<8 +} + +func reverseBits(number uint16, bitLength byte) uint16 { + return reverseUint16(number << uint8(16-bitLength)) +} diff --git a/libgo/go/compress/flate/token.go b/libgo/go/compress/flate/token.go new file mode 100644 index 000000000..38aea5fa6 --- /dev/null +++ b/libgo/go/compress/flate/token.go @@ -0,0 +1,103 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package flate + +const ( + // 2 bits: type 0 = literal 1=EOF 2=Match 3=Unused + // 8 bits: xlength = length - MIN_MATCH_LENGTH + // 22 bits xoffset = offset - MIN_OFFSET_SIZE, or literal + lengthShift = 22 + offsetMask = 1< pair into a match token. +func matchToken(xlength uint32, xoffset uint32) token { + return token(matchType + xlength<> lengthShift) } + +func lengthCode(len uint32) uint32 { return lengthCodes[len] } + +// Returns the offset code corresponding to a specific offset +func offsetCode(off uint32) uint32 { + const n = uint32(len(offsetCodes)) + switch { + case off < n: + return offsetCodes[off] + case off>>7 < n: + return offsetCodes[off>>7] + 14 + default: + return offsetCodes[off>>14] + 28 + } + panic("unreachable") +} diff --git a/libgo/go/compress/flate/util.go b/libgo/go/compress/flate/util.go new file mode 100644 index 000000000..aca5c78b2 --- /dev/null +++ b/libgo/go/compress/flate/util.go @@ -0,0 +1,72 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package flate + +func min(left int, right int) int { + if left < right { + return left + } + return right +} + +func minInt32(left int32, right int32) int32 { + if left < right { + return left + } + return right +} + +func max(left int, right int) int { + if left > right { + return left + } + return right +} + +func fillInts(a []int, value int) { + for i := range a { + a[i] = value + } +} + +func fillInt32s(a []int32, value int32) { + for i := range a { + a[i] = value + } +} + +func fillBytes(a []byte, value byte) { + for i := range a { + a[i] = value + } +} + +func fillInt8s(a []int8, value int8) { + for i := range a { + a[i] = value + } +} + +func fillUint8s(a []uint8, value uint8) { + for i := range a { + a[i] = value + } +} + +func copyInt8s(dst []int8, src []int8) int { + cnt := min(len(dst), len(src)) + for i := 0; i < cnt; i++ { + dst[i] = src[i] + } + return cnt +} + +func copyUint8s(dst []uint8, src []uint8) int { + cnt := min(len(dst), len(src)) + for i := 0; i < cnt; i++ { + dst[i] = src[i] + } + return cnt +} diff --git a/libgo/go/compress/gzip/gunzip.go b/libgo/go/compress/gzip/gunzip.go new file mode 100644 index 000000000..3c0b3c5e5 --- /dev/null +++ b/libgo/go/compress/gzip/gunzip.go @@ -0,0 +1,230 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The gzip package implements reading and writing of +// gzip format compressed files, as specified in RFC 1952. +package gzip + +import ( + "bufio" + "compress/flate" + "hash" + "hash/crc32" + "io" + "os" +) + +// BUG(nigeltao): Comments and Names don't properly map UTF-8 character codes outside of +// the 0x00-0x7f range to ISO 8859-1 (Latin-1). + +const ( + gzipID1 = 0x1f + gzipID2 = 0x8b + gzipDeflate = 8 + flagText = 1 << 0 + flagHdrCrc = 1 << 1 + flagExtra = 1 << 2 + flagName = 1 << 3 + flagComment = 1 << 4 +) + +func makeReader(r io.Reader) flate.Reader { + if rr, ok := r.(flate.Reader); ok { + return rr + } + return bufio.NewReader(r) +} + +var HeaderError os.Error = os.ErrorString("invalid gzip header") +var ChecksumError os.Error = os.ErrorString("gzip checksum error") + +// The gzip file stores a header giving metadata about the compressed file. +// That header is exposed as the fields of the Compressor and Decompressor structs. +type Header struct { + Comment string // comment + Extra []byte // "extra data" + Mtime uint32 // modification time (seconds since January 1, 1970) + Name string // file name + OS byte // operating system type +} + +// An Decompressor is an io.Reader that can be read to retrieve +// uncompressed data from a gzip-format compressed file. +// +// In general, a gzip file can be a concatenation of gzip files, +// each with its own header. Reads from the Decompressor +// return the concatenation of the uncompressed data of each. +// Only the first header is recorded in the Decompressor fields. +// +// Gzip files store a length and checksum of the uncompressed data. +// The Decompressor will return a ChecksumError when Read +// reaches the end of the uncompressed data if it does not +// have the expected length or checksum. Clients should treat data +// returned by Read as tentative until they receive the successful +// (zero length, nil error) Read marking the end of the data. +type Decompressor struct { + Header + r flate.Reader + decompressor io.ReadCloser + digest hash.Hash32 + size uint32 + flg byte + buf [512]byte + err os.Error +} + +// NewReader creates a new Decompressor reading the given reader. +// The implementation buffers input and may read more data than necessary from r. +// It is the caller's responsibility to call Close on the Decompressor when done. +func NewReader(r io.Reader) (*Decompressor, os.Error) { + z := new(Decompressor) + z.r = makeReader(r) + z.digest = crc32.NewIEEE() + if err := z.readHeader(true); err != nil { + z.err = err + return nil, err + } + return z, nil +} + +// GZIP (RFC 1952) is little-endian, unlike ZLIB (RFC 1950). +func get4(p []byte) uint32 { + return uint32(p[0]) | uint32(p[1])<<8 | uint32(p[2])<<16 | uint32(p[3])<<24 +} + +func (z *Decompressor) readString() (string, os.Error) { + var err os.Error + for i := 0; ; i++ { + if i >= len(z.buf) { + return "", HeaderError + } + z.buf[i], err = z.r.ReadByte() + if err != nil { + return "", err + } + if z.buf[i] == 0 { + // GZIP (RFC 1952) specifies that strings are NUL-terminated ISO 8859-1 (Latin-1). + // TODO(nigeltao): Convert from ISO 8859-1 (Latin-1) to UTF-8. + return string(z.buf[0:i]), nil + } + } + panic("not reached") +} + +func (z *Decompressor) read2() (uint32, os.Error) { + _, err := io.ReadFull(z.r, z.buf[0:2]) + if err != nil { + return 0, err + } + return uint32(z.buf[0]) | uint32(z.buf[1])<<8, nil +} + +func (z *Decompressor) readHeader(save bool) os.Error { + _, err := io.ReadFull(z.r, z.buf[0:10]) + if err != nil { + return err + } + if z.buf[0] != gzipID1 || z.buf[1] != gzipID2 || z.buf[2] != gzipDeflate { + return HeaderError + } + z.flg = z.buf[3] + if save { + z.Mtime = get4(z.buf[4:8]) + // z.buf[8] is xfl, ignored + z.OS = z.buf[9] + } + z.digest.Reset() + z.digest.Write(z.buf[0:10]) + + if z.flg&flagExtra != 0 { + n, err := z.read2() + if err != nil { + return err + } + data := make([]byte, n) + if _, err = io.ReadFull(z.r, data); err != nil { + return err + } + if save { + z.Extra = data + } + } + + var s string + if z.flg&flagName != 0 { + if s, err = z.readString(); err != nil { + return err + } + if save { + z.Name = s + } + } + + if z.flg&flagComment != 0 { + if s, err = z.readString(); err != nil { + return err + } + if save { + z.Comment = s + } + } + + if z.flg&flagHdrCrc != 0 { + n, err := z.read2() + if err != nil { + return err + } + sum := z.digest.Sum32() & 0xFFFF + if n != sum { + return HeaderError + } + } + + z.digest.Reset() + z.decompressor = flate.NewReader(z.r) + return nil +} + +func (z *Decompressor) Read(p []byte) (n int, err os.Error) { + if z.err != nil { + return 0, z.err + } + if len(p) == 0 { + return 0, nil + } + + n, err = z.decompressor.Read(p) + z.digest.Write(p[0:n]) + z.size += uint32(n) + if n != 0 || err != os.EOF { + z.err = err + return + } + + // Finished file; check checksum + size. + if _, err := io.ReadFull(z.r, z.buf[0:8]); err != nil { + z.err = err + return 0, err + } + crc32, isize := get4(z.buf[0:4]), get4(z.buf[4:8]) + sum := z.digest.Sum32() + if sum != crc32 || isize != z.size { + z.err = ChecksumError + return 0, z.err + } + + // File is ok; is there another? + if err = z.readHeader(false); err != nil { + z.err = err + return + } + + // Yes. Reset and read from it. + z.digest.Reset() + z.size = 0 + return z.Read(p) +} + +// Calling Close does not close the wrapped io.Reader originally passed to NewReader. +func (z *Decompressor) Close() os.Error { return z.decompressor.Close() } diff --git a/libgo/go/compress/gzip/gunzip_test.go b/libgo/go/compress/gzip/gunzip_test.go new file mode 100644 index 000000000..1c08c7374 --- /dev/null +++ b/libgo/go/compress/gzip/gunzip_test.go @@ -0,0 +1,305 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gzip + +import ( + "bytes" + "io" + "os" + "testing" +) + +type gunzipTest struct { + name string + desc string + raw string + gzip []byte + err os.Error +} + +var gunzipTests = []gunzipTest{ + { // has 1 empty fixed-huffman block + "empty.txt", + "empty.txt", + "", + []byte{ + 0x1f, 0x8b, 0x08, 0x08, 0xf7, 0x5e, 0x14, 0x4a, + 0x00, 0x03, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2e, + 0x74, 0x78, 0x74, 0x00, 0x03, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + }, + nil, + }, + { // has 1 non-empty fixed huffman block + "hello.txt", + "hello.txt", + "hello world\n", + []byte{ + 0x1f, 0x8b, 0x08, 0x08, 0xc8, 0x58, 0x13, 0x4a, + 0x00, 0x03, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x2e, + 0x74, 0x78, 0x74, 0x00, 0xcb, 0x48, 0xcd, 0xc9, + 0xc9, 0x57, 0x28, 0xcf, 0x2f, 0xca, 0x49, 0xe1, + 0x02, 0x00, 0x2d, 0x3b, 0x08, 0xaf, 0x0c, 0x00, + 0x00, 0x00, + }, + nil, + }, + { // concatenation + "hello.txt", + "hello.txt x2", + "hello world\n" + + "hello world\n", + []byte{ + 0x1f, 0x8b, 0x08, 0x08, 0xc8, 0x58, 0x13, 0x4a, + 0x00, 0x03, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x2e, + 0x74, 0x78, 0x74, 0x00, 0xcb, 0x48, 0xcd, 0xc9, + 0xc9, 0x57, 0x28, 0xcf, 0x2f, 0xca, 0x49, 0xe1, + 0x02, 0x00, 0x2d, 0x3b, 0x08, 0xaf, 0x0c, 0x00, + 0x00, 0x00, + 0x1f, 0x8b, 0x08, 0x08, 0xc8, 0x58, 0x13, 0x4a, + 0x00, 0x03, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x2e, + 0x74, 0x78, 0x74, 0x00, 0xcb, 0x48, 0xcd, 0xc9, + 0xc9, 0x57, 0x28, 0xcf, 0x2f, 0xca, 0x49, 0xe1, + 0x02, 0x00, 0x2d, 0x3b, 0x08, 0xaf, 0x0c, 0x00, + 0x00, 0x00, + }, + nil, + }, + { // has a fixed huffman block with some length-distance pairs + "shesells.txt", + "shesells.txt", + "she sells seashells by the seashore\n", + []byte{ + 0x1f, 0x8b, 0x08, 0x08, 0x72, 0x66, 0x8b, 0x4a, + 0x00, 0x03, 0x73, 0x68, 0x65, 0x73, 0x65, 0x6c, + 0x6c, 0x73, 0x2e, 0x74, 0x78, 0x74, 0x00, 0x2b, + 0xce, 0x48, 0x55, 0x28, 0x4e, 0xcd, 0xc9, 0x29, + 0x06, 0x92, 0x89, 0xc5, 0x19, 0x60, 0x56, 0x52, + 0xa5, 0x42, 0x09, 0x58, 0x18, 0x28, 0x90, 0x5f, + 0x94, 0xca, 0x05, 0x00, 0x76, 0xb0, 0x3b, 0xeb, + 0x24, 0x00, 0x00, 0x00, + }, + nil, + }, + { // has dynamic huffman blocks + "gettysburg", + "gettysburg", + " Four score and seven years ago our fathers brought forth on\n" + + "this continent, a new nation, conceived in Liberty, and dedicated\n" + + "to the proposition that all men are created equal.\n" + + " Now we are engaged in a great Civil War, testing whether that\n" + + "nation, or any nation so conceived and so dedicated, can long\n" + + "endure.\n" + + " We are met on a great battle-field of that war.\n" + + " We have come to dedicate a portion of that field, as a final\n" + + "resting place for those who here gave their lives that that\n" + + "nation might live. It is altogether fitting and proper that\n" + + "we should do this.\n" + + " But, in a larger sense, we can not dedicate — we can not\n" + + "consecrate — we can not hallow — this ground.\n" + + " The brave men, living and dead, who struggled here, have\n" + + "consecrated it, far above our poor power to add or detract.\n" + + "The world will little note, nor long remember what we say here,\n" + + "but it can never forget what they did here.\n" + + " It is for us the living, rather, to be dedicated here to the\n" + + "unfinished work which they who fought here have thus far so\n" + + "nobly advanced. It is rather for us to be here dedicated to\n" + + "the great task remaining before us — that from these honored\n" + + "dead we take increased devotion to that cause for which they\n" + + "gave the last full measure of devotion —\n" + + " that we here highly resolve that these dead shall not have\n" + + "died in vain — that this nation, under God, shall have a new\n" + + "birth of freedom — and that government of the people, by the\n" + + "people, for the people, shall not perish from this earth.\n" + + "\n" + + "Abraham Lincoln, November 19, 1863, Gettysburg, Pennsylvania\n", + []byte{ + 0x1f, 0x8b, 0x08, 0x08, 0xd1, 0x12, 0x2b, 0x4a, + 0x00, 0x03, 0x67, 0x65, 0x74, 0x74, 0x79, 0x73, + 0x62, 0x75, 0x72, 0x67, 0x00, 0x65, 0x54, 0xcd, + 0x6e, 0xd4, 0x30, 0x10, 0xbe, 0xfb, 0x29, 0xe6, + 0x01, 0x42, 0xa5, 0x0a, 0x09, 0xc1, 0x11, 0x90, + 0x40, 0x48, 0xa8, 0xe2, 0x80, 0xd4, 0xf3, 0x24, + 0x9e, 0x24, 0x56, 0xbd, 0x9e, 0xc5, 0x76, 0x76, + 0x95, 0x1b, 0x0f, 0xc1, 0x13, 0xf2, 0x24, 0x7c, + 0x63, 0x77, 0x9b, 0x4a, 0x5c, 0xaa, 0x6e, 0x6c, + 0xcf, 0x7c, 0x7f, 0x33, 0x44, 0x5f, 0x74, 0xcb, + 0x54, 0x26, 0xcd, 0x42, 0x9c, 0x3c, 0x15, 0xb9, + 0x48, 0xa2, 0x5d, 0x38, 0x17, 0xe2, 0x45, 0xc9, + 0x4e, 0x67, 0xae, 0xab, 0xe0, 0xf7, 0x98, 0x75, + 0x5b, 0xd6, 0x4a, 0xb3, 0xe6, 0xba, 0x92, 0x26, + 0x57, 0xd7, 0x50, 0x68, 0xd2, 0x54, 0x43, 0x92, + 0x54, 0x07, 0x62, 0x4a, 0x72, 0xa5, 0xc4, 0x35, + 0x68, 0x1a, 0xec, 0x60, 0x92, 0x70, 0x11, 0x4f, + 0x21, 0xd1, 0xf7, 0x30, 0x4a, 0xae, 0xfb, 0xd0, + 0x9a, 0x78, 0xf1, 0x61, 0xe2, 0x2a, 0xde, 0x55, + 0x25, 0xd4, 0xa6, 0x73, 0xd6, 0xb3, 0x96, 0x60, + 0xef, 0xf0, 0x9b, 0x2b, 0x71, 0x8c, 0x74, 0x02, + 0x10, 0x06, 0xac, 0x29, 0x8b, 0xdd, 0x25, 0xf9, + 0xb5, 0x71, 0xbc, 0x73, 0x44, 0x0f, 0x7a, 0xa5, + 0xab, 0xb4, 0x33, 0x49, 0x0b, 0x2f, 0xbd, 0x03, + 0xd3, 0x62, 0x17, 0xe9, 0x73, 0xb8, 0x84, 0x48, + 0x8f, 0x9c, 0x07, 0xaa, 0x52, 0x00, 0x6d, 0xa1, + 0xeb, 0x2a, 0xc6, 0xa0, 0x95, 0x76, 0x37, 0x78, + 0x9a, 0x81, 0x65, 0x7f, 0x46, 0x4b, 0x45, 0x5f, + 0xe1, 0x6d, 0x42, 0xe8, 0x01, 0x13, 0x5c, 0x38, + 0x51, 0xd4, 0xb4, 0x38, 0x49, 0x7e, 0xcb, 0x62, + 0x28, 0x1e, 0x3b, 0x82, 0x93, 0x54, 0x48, 0xf1, + 0xd2, 0x7d, 0xe4, 0x5a, 0xa3, 0xbc, 0x99, 0x83, + 0x44, 0x4f, 0x3a, 0x77, 0x36, 0x57, 0xce, 0xcf, + 0x2f, 0x56, 0xbe, 0x80, 0x90, 0x9e, 0x84, 0xea, + 0x51, 0x1f, 0x8f, 0xcf, 0x90, 0xd4, 0x60, 0xdc, + 0x5e, 0xb4, 0xf7, 0x10, 0x0b, 0x26, 0xe0, 0xff, + 0xc4, 0xd1, 0xe5, 0x67, 0x2e, 0xe7, 0xc8, 0x93, + 0x98, 0x05, 0xb8, 0xa8, 0x45, 0xc0, 0x4d, 0x09, + 0xdc, 0x84, 0x16, 0x2b, 0x0d, 0x9a, 0x21, 0x53, + 0x04, 0x8b, 0xd2, 0x0b, 0xbd, 0xa2, 0x4c, 0xa7, + 0x60, 0xee, 0xd9, 0xe1, 0x1d, 0xd1, 0xb7, 0x4a, + 0x30, 0x8f, 0x63, 0xd5, 0xa5, 0x8b, 0x33, 0x87, + 0xda, 0x1a, 0x18, 0x79, 0xf3, 0xe3, 0xa6, 0x17, + 0x94, 0x2e, 0xab, 0x6e, 0xa0, 0xe3, 0xcd, 0xac, + 0x50, 0x8c, 0xca, 0xa7, 0x0d, 0x76, 0x37, 0xd1, + 0x23, 0xe7, 0x05, 0x57, 0x8b, 0xa4, 0x22, 0x83, + 0xd9, 0x62, 0x52, 0x25, 0xad, 0x07, 0xbb, 0xbf, + 0xbf, 0xff, 0xbc, 0xfa, 0xee, 0x20, 0x73, 0x91, + 0x29, 0xff, 0x7f, 0x02, 0x71, 0x62, 0x84, 0xb5, + 0xf6, 0xb5, 0x25, 0x6b, 0x41, 0xde, 0x92, 0xb7, + 0x76, 0x3f, 0x91, 0x91, 0x31, 0x1b, 0x41, 0x84, + 0x62, 0x30, 0x0a, 0x37, 0xa4, 0x5e, 0x18, 0x3a, + 0x99, 0x08, 0xa5, 0xe6, 0x6d, 0x59, 0x22, 0xec, + 0x33, 0x39, 0x86, 0x26, 0xf5, 0xab, 0x66, 0xc8, + 0x08, 0x20, 0xcf, 0x0c, 0xd7, 0x47, 0x45, 0x21, + 0x0b, 0xf6, 0x59, 0xd5, 0xfe, 0x5c, 0x8d, 0xaa, + 0x12, 0x7b, 0x6f, 0xa1, 0xf0, 0x52, 0x33, 0x4f, + 0xf5, 0xce, 0x59, 0xd3, 0xab, 0x66, 0x10, 0xbf, + 0x06, 0xc4, 0x31, 0x06, 0x73, 0xd6, 0x80, 0xa2, + 0x78, 0xc2, 0x45, 0xcb, 0x03, 0x65, 0x39, 0xc9, + 0x09, 0xd1, 0x06, 0x04, 0x33, 0x1a, 0x5a, 0xf1, + 0xde, 0x01, 0xb8, 0x71, 0x83, 0xc4, 0xb5, 0xb3, + 0xc3, 0x54, 0x65, 0x33, 0x0d, 0x5a, 0xf7, 0x9b, + 0x90, 0x7c, 0x27, 0x1f, 0x3a, 0x58, 0xa3, 0xd8, + 0xfd, 0x30, 0x5f, 0xb7, 0xd2, 0x66, 0xa2, 0x93, + 0x1c, 0x28, 0xb7, 0xe9, 0x1b, 0x0c, 0xe1, 0x28, + 0x47, 0x26, 0xbb, 0xe9, 0x7d, 0x7e, 0xdc, 0x96, + 0x10, 0x92, 0x50, 0x56, 0x7c, 0x06, 0xe2, 0x27, + 0xb4, 0x08, 0xd3, 0xda, 0x7b, 0x98, 0x34, 0x73, + 0x9f, 0xdb, 0xf6, 0x62, 0xed, 0x31, 0x41, 0x13, + 0xd3, 0xa2, 0xa8, 0x4b, 0x3a, 0xc6, 0x1d, 0xe4, + 0x2f, 0x8c, 0xf8, 0xfb, 0x97, 0x64, 0xf4, 0xb6, + 0x2f, 0x80, 0x5a, 0xf3, 0x56, 0xe0, 0x40, 0x50, + 0xd5, 0x19, 0xd0, 0x1e, 0xfc, 0xca, 0xe5, 0xc9, + 0xd4, 0x60, 0x00, 0x81, 0x2e, 0xa3, 0xcc, 0xb6, + 0x52, 0xf0, 0xb4, 0xdb, 0x69, 0x99, 0xce, 0x7a, + 0x32, 0x4c, 0x08, 0xed, 0xaa, 0x10, 0x10, 0xe3, + 0x6f, 0xee, 0x99, 0x68, 0x95, 0x9f, 0x04, 0x71, + 0xb2, 0x49, 0x2f, 0x62, 0xa6, 0x5e, 0xb4, 0xef, + 0x02, 0xed, 0x4f, 0x27, 0xde, 0x4a, 0x0f, 0xfd, + 0xc1, 0xcc, 0xdd, 0x02, 0x8f, 0x08, 0x16, 0x54, + 0xdf, 0xda, 0xca, 0xe0, 0x82, 0xf1, 0xb4, 0x31, + 0x7a, 0xa9, 0x81, 0xfe, 0x90, 0xb7, 0x3e, 0xdb, + 0xd3, 0x35, 0xc0, 0x20, 0x80, 0x33, 0x46, 0x4a, + 0x63, 0xab, 0xd1, 0x0d, 0x29, 0xd2, 0xe2, 0x84, + 0xb8, 0xdb, 0xfa, 0xe9, 0x89, 0x44, 0x86, 0x7c, + 0xe8, 0x0b, 0xe6, 0x02, 0x6a, 0x07, 0x9b, 0x96, + 0xd0, 0xdb, 0x2e, 0x41, 0x4c, 0xa1, 0xd5, 0x57, + 0x45, 0x14, 0xfb, 0xe3, 0xa6, 0x72, 0x5b, 0x87, + 0x6e, 0x0c, 0x6d, 0x5b, 0xce, 0xe0, 0x2f, 0xe2, + 0x21, 0x81, 0x95, 0xb0, 0xe8, 0xb6, 0x32, 0x0b, + 0xb2, 0x98, 0x13, 0x52, 0x5d, 0xfb, 0xec, 0x63, + 0x17, 0x8a, 0x9e, 0x23, 0x22, 0x36, 0xee, 0xcd, + 0xda, 0xdb, 0xcf, 0x3e, 0xf1, 0xc7, 0xf1, 0x01, + 0x12, 0x93, 0x0a, 0xeb, 0x6f, 0xf2, 0x02, 0x15, + 0x96, 0x77, 0x5d, 0xef, 0x9c, 0xfb, 0x88, 0x91, + 0x59, 0xf9, 0x84, 0xdd, 0x9b, 0x26, 0x8d, 0x80, + 0xf9, 0x80, 0x66, 0x2d, 0xac, 0xf7, 0x1f, 0x06, + 0xba, 0x7f, 0xff, 0xee, 0xed, 0x40, 0x5f, 0xa5, + 0xd6, 0xbd, 0x8c, 0x5b, 0x46, 0xd2, 0x7e, 0x48, + 0x4a, 0x65, 0x8f, 0x08, 0x42, 0x60, 0xf7, 0x0f, + 0xb9, 0x16, 0x0b, 0x0c, 0x1a, 0x06, 0x00, 0x00, + }, + nil, + }, + { // has 1 non-empty fixed huffman block then garbage + "hello.txt", + "hello.txt + garbage", + "hello world\n", + []byte{ + 0x1f, 0x8b, 0x08, 0x08, 0xc8, 0x58, 0x13, 0x4a, + 0x00, 0x03, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x2e, + 0x74, 0x78, 0x74, 0x00, 0xcb, 0x48, 0xcd, 0xc9, + 0xc9, 0x57, 0x28, 0xcf, 0x2f, 0xca, 0x49, 0xe1, + 0x02, 0x00, 0x2d, 0x3b, 0x08, 0xaf, 0x0c, 0x00, + 0x00, 0x00, 'g', 'a', 'r', 'b', 'a', 'g', 'e', '!', '!', '!', + }, + HeaderError, + }, + { // has 1 non-empty fixed huffman block not enough header + "hello.txt", + "hello.txt + garbage", + "hello world\n", + []byte{ + 0x1f, 0x8b, 0x08, 0x08, 0xc8, 0x58, 0x13, 0x4a, + 0x00, 0x03, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x2e, + 0x74, 0x78, 0x74, 0x00, 0xcb, 0x48, 0xcd, 0xc9, + 0xc9, 0x57, 0x28, 0xcf, 0x2f, 0xca, 0x49, 0xe1, + 0x02, 0x00, 0x2d, 0x3b, 0x08, 0xaf, 0x0c, 0x00, + 0x00, 0x00, gzipID1, + }, + io.ErrUnexpectedEOF, + }, + { // has 1 non-empty fixed huffman block but corrupt checksum + "hello.txt", + "hello.txt + corrupt checksum", + "hello world\n", + []byte{ + 0x1f, 0x8b, 0x08, 0x08, 0xc8, 0x58, 0x13, 0x4a, + 0x00, 0x03, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x2e, + 0x74, 0x78, 0x74, 0x00, 0xcb, 0x48, 0xcd, 0xc9, + 0xc9, 0x57, 0x28, 0xcf, 0x2f, 0xca, 0x49, 0xe1, + 0x02, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0c, 0x00, + 0x00, 0x00, + }, + ChecksumError, + }, + { // has 1 non-empty fixed huffman block but corrupt size + "hello.txt", + "hello.txt + corrupt size", + "hello world\n", + []byte{ + 0x1f, 0x8b, 0x08, 0x08, 0xc8, 0x58, 0x13, 0x4a, + 0x00, 0x03, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x2e, + 0x74, 0x78, 0x74, 0x00, 0xcb, 0x48, 0xcd, 0xc9, + 0xc9, 0x57, 0x28, 0xcf, 0x2f, 0xca, 0x49, 0xe1, + 0x02, 0x00, 0x2d, 0x3b, 0x08, 0xaf, 0xff, 0x00, + 0x00, 0x00, + }, + ChecksumError, + }, +} + +func TestDecompressor(t *testing.T) { + b := new(bytes.Buffer) + for _, tt := range gunzipTests { + in := bytes.NewBuffer(tt.gzip) + gzip, err := NewReader(in) + if err != nil { + t.Errorf("%s: NewReader: %s", tt.name, err) + continue + } + defer gzip.Close() + if tt.name != gzip.Name { + t.Errorf("%s: got name %s", tt.name, gzip.Name) + } + b.Reset() + n, err := io.Copy(b, gzip) + if err != tt.err { + t.Errorf("%s: io.Copy: %v want %v", tt.name, err, tt.err) + } + s := b.String() + if s != tt.raw { + t.Errorf("%s: got %d-byte %q want %d-byte %q", tt.name, n, s, len(tt.raw), tt.raw) + } + } +} diff --git a/libgo/go/compress/gzip/gzip.go b/libgo/go/compress/gzip/gzip.go new file mode 100644 index 000000000..8860d10af --- /dev/null +++ b/libgo/go/compress/gzip/gzip.go @@ -0,0 +1,187 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gzip + +import ( + "compress/flate" + "hash" + "hash/crc32" + "io" + "os" +) + +// These constants are copied from the flate package, so that code that imports +// "compress/gzip" does not also have to import "compress/flate". +const ( + NoCompression = flate.NoCompression + BestSpeed = flate.BestSpeed + BestCompression = flate.BestCompression + DefaultCompression = flate.DefaultCompression +) + +// A Compressor is an io.WriteCloser that satisfies writes by compressing data written +// to its wrapped io.Writer. +type Compressor struct { + Header + w io.Writer + level int + compressor io.WriteCloser + digest hash.Hash32 + size uint32 + closed bool + buf [10]byte + err os.Error +} + +// NewWriter calls NewWriterLevel with the default compression level. +func NewWriter(w io.Writer) (*Compressor, os.Error) { + return NewWriterLevel(w, DefaultCompression) +} + +// NewWriterLevel creates a new Compressor writing to the given writer. +// Writes may be buffered and not flushed until Close. +// Callers that wish to set the fields in Compressor.Header must +// do so before the first call to Write or Close. +// It is the caller's responsibility to call Close on the WriteCloser when done. +// level is the compression level, which can be DefaultCompression, NoCompression, +// or any integer value between BestSpeed and BestCompression (inclusive). +func NewWriterLevel(w io.Writer, level int) (*Compressor, os.Error) { + z := new(Compressor) + z.OS = 255 // unknown + z.w = w + z.level = level + z.digest = crc32.NewIEEE() + return z, nil +} + +// GZIP (RFC 1952) is little-endian, unlike ZLIB (RFC 1950). +func put2(p []byte, v uint16) { + p[0] = uint8(v >> 0) + p[1] = uint8(v >> 8) +} + +func put4(p []byte, v uint32) { + p[0] = uint8(v >> 0) + p[1] = uint8(v >> 8) + p[2] = uint8(v >> 16) + p[3] = uint8(v >> 24) +} + +// writeBytes writes a length-prefixed byte slice to z.w. +func (z *Compressor) writeBytes(b []byte) os.Error { + if len(b) > 0xffff { + return os.NewError("gzip.Write: Extra data is too large") + } + put2(z.buf[0:2], uint16(len(b))) + _, err := z.w.Write(z.buf[0:2]) + if err != nil { + return err + } + _, err = z.w.Write(b) + return err +} + +// writeString writes a string (in ISO 8859-1 (Latin-1) format) to z.w. +func (z *Compressor) writeString(s string) os.Error { + // GZIP (RFC 1952) specifies that strings are NUL-terminated ISO 8859-1 (Latin-1). + // TODO(nigeltao): Convert from UTF-8 to ISO 8859-1 (Latin-1). + for _, v := range s { + if v == 0 || v > 0x7f { + return os.NewError("gzip.Write: non-ASCII header string") + } + } + _, err := io.WriteString(z.w, s) + if err != nil { + return err + } + // GZIP strings are NUL-terminated. + z.buf[0] = 0 + _, err = z.w.Write(z.buf[0:1]) + return err +} + +func (z *Compressor) Write(p []byte) (int, os.Error) { + if z.err != nil { + return 0, z.err + } + var n int + // Write the GZIP header lazily. + if z.compressor == nil { + z.buf[0] = gzipID1 + z.buf[1] = gzipID2 + z.buf[2] = gzipDeflate + z.buf[3] = 0 + if z.Extra != nil { + z.buf[3] |= 0x04 + } + if z.Name != "" { + z.buf[3] |= 0x08 + } + if z.Comment != "" { + z.buf[3] |= 0x10 + } + put4(z.buf[4:8], z.Mtime) + if z.level == BestCompression { + z.buf[8] = 2 + } else if z.level == BestSpeed { + z.buf[8] = 4 + } else { + z.buf[8] = 0 + } + z.buf[9] = z.OS + n, z.err = z.w.Write(z.buf[0:10]) + if z.err != nil { + return n, z.err + } + if z.Extra != nil { + z.err = z.writeBytes(z.Extra) + if z.err != nil { + return n, z.err + } + } + if z.Name != "" { + z.err = z.writeString(z.Name) + if z.err != nil { + return n, z.err + } + } + if z.Comment != "" { + z.err = z.writeString(z.Comment) + if z.err != nil { + return n, z.err + } + } + z.compressor = flate.NewWriter(z.w, z.level) + } + z.size += uint32(len(p)) + z.digest.Write(p) + n, z.err = z.compressor.Write(p) + return n, z.err +} + +// Calling Close does not close the wrapped io.Writer originally passed to NewWriter. +func (z *Compressor) Close() os.Error { + if z.err != nil { + return z.err + } + if z.closed { + return nil + } + z.closed = true + if z.compressor == nil { + z.Write(nil) + if z.err != nil { + return z.err + } + } + z.err = z.compressor.Close() + if z.err != nil { + return z.err + } + put4(z.buf[0:4], z.digest.Sum32()) + put4(z.buf[4:8], z.size) + _, z.err = z.w.Write(z.buf[0:8]) + return z.err +} diff --git a/libgo/go/compress/gzip/gzip_test.go b/libgo/go/compress/gzip/gzip_test.go new file mode 100644 index 000000000..23f351405 --- /dev/null +++ b/libgo/go/compress/gzip/gzip_test.go @@ -0,0 +1,84 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gzip + +import ( + "io" + "io/ioutil" + "testing" +) + +// pipe creates two ends of a pipe that gzip and gunzip, and runs dfunc at the +// writer end and ifunc at the reader end. +func pipe(t *testing.T, dfunc func(*Compressor), cfunc func(*Decompressor)) { + piper, pipew := io.Pipe() + defer piper.Close() + go func() { + defer pipew.Close() + compressor, err := NewWriter(pipew) + if err != nil { + t.Fatalf("%v", err) + } + defer compressor.Close() + dfunc(compressor) + }() + decompressor, err := NewReader(piper) + if err != nil { + t.Fatalf("%v", err) + } + defer decompressor.Close() + cfunc(decompressor) +} + +// Tests that an empty payload still forms a valid GZIP stream. +func TestEmpty(t *testing.T) { + pipe(t, + func(compressor *Compressor) {}, + func(decompressor *Decompressor) { + b, err := ioutil.ReadAll(decompressor) + if err != nil { + t.Fatalf("%v", err) + } + if len(b) != 0 { + t.Fatalf("did not read an empty slice") + } + }) +} + +// Tests that gzipping and then gunzipping is the identity function. +func TestWriter(t *testing.T) { + pipe(t, + func(compressor *Compressor) { + compressor.Comment = "comment" + compressor.Extra = []byte("extra") + compressor.Mtime = 1e8 + compressor.Name = "name" + _, err := compressor.Write([]byte("payload")) + if err != nil { + t.Fatalf("%v", err) + } + }, + func(decompressor *Decompressor) { + b, err := ioutil.ReadAll(decompressor) + if err != nil { + t.Fatalf("%v", err) + } + if string(b) != "payload" { + t.Fatalf("payload is %q, want %q", string(b), "payload") + } + if decompressor.Comment != "comment" { + t.Fatalf("comment is %q, want %q", decompressor.Comment, "comment") + } + if string(decompressor.Extra) != "extra" { + t.Fatalf("extra is %q, want %q", decompressor.Extra, "extra") + } + if decompressor.Mtime != 1e8 { + t.Fatalf("mtime is %d, want %d", decompressor.Mtime, uint32(1e8)) + } + if decompressor.Name != "name" { + t.Fatalf("name is %q, want %q", decompressor.Name, "name") + } + }) +} diff --git a/libgo/go/compress/zlib/reader.go b/libgo/go/compress/zlib/reader.go new file mode 100644 index 000000000..721f6ec55 --- /dev/null +++ b/libgo/go/compress/zlib/reader.go @@ -0,0 +1,112 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +The zlib package implements reading and writing of zlib +format compressed data, as specified in RFC 1950. + +The implementation provides filters that uncompress during reading +and compress during writing. For example, to write compressed data +to a buffer: + + var b bytes.Buffer + w, err := zlib.NewWriter(&b) + w.Write([]byte("hello, world\n")) + w.Close() + +and to read that data back: + + r, err := zlib.NewReader(&b) + io.Copy(os.Stdout, r) + r.Close() +*/ +package zlib + +import ( + "bufio" + "compress/flate" + "hash" + "hash/adler32" + "io" + "os" +) + +const zlibDeflate = 8 + +var ChecksumError os.Error = os.ErrorString("zlib checksum error") +var HeaderError os.Error = os.ErrorString("invalid zlib header") +var UnsupportedError os.Error = os.ErrorString("unsupported zlib format") + +type reader struct { + r flate.Reader + decompressor io.ReadCloser + digest hash.Hash32 + err os.Error + scratch [4]byte +} + +// NewReader creates a new io.ReadCloser that satisfies reads by decompressing data read from r. +// The implementation buffers input and may read more data than necessary from r. +// It is the caller's responsibility to call Close on the ReadCloser when done. +func NewReader(r io.Reader) (io.ReadCloser, os.Error) { + z := new(reader) + if fr, ok := r.(flate.Reader); ok { + z.r = fr + } else { + z.r = bufio.NewReader(r) + } + _, err := io.ReadFull(z.r, z.scratch[0:2]) + if err != nil { + return nil, err + } + h := uint(z.scratch[0])<<8 | uint(z.scratch[1]) + if (z.scratch[0]&0x0f != zlibDeflate) || (h%31 != 0) { + return nil, HeaderError + } + if z.scratch[1]&0x20 != 0 { + // BUG(nigeltao): The zlib package does not implement the FDICT flag. + return nil, UnsupportedError + } + z.digest = adler32.New() + z.decompressor = flate.NewReader(z.r) + return z, nil +} + +func (z *reader) Read(p []byte) (n int, err os.Error) { + if z.err != nil { + return 0, z.err + } + if len(p) == 0 { + return 0, nil + } + + n, err = z.decompressor.Read(p) + z.digest.Write(p[0:n]) + if n != 0 || err != os.EOF { + z.err = err + return + } + + // Finished file; check checksum. + if _, err := io.ReadFull(z.r, z.scratch[0:4]); err != nil { + z.err = err + return 0, err + } + // ZLIB (RFC 1950) is big-endian, unlike GZIP (RFC 1952). + checksum := uint32(z.scratch[0])<<24 | uint32(z.scratch[1])<<16 | uint32(z.scratch[2])<<8 | uint32(z.scratch[3]) + if checksum != z.digest.Sum32() { + z.err = ChecksumError + return 0, z.err + } + return +} + +// Calling Close does not close the wrapped io.Reader originally passed to NewReader. +func (z *reader) Close() os.Error { + if z.err != nil { + return z.err + } + z.err = z.decompressor.Close() + return z.err +} diff --git a/libgo/go/compress/zlib/reader_test.go b/libgo/go/compress/zlib/reader_test.go new file mode 100644 index 000000000..eaefc3a36 --- /dev/null +++ b/libgo/go/compress/zlib/reader_test.go @@ -0,0 +1,95 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package zlib + +import ( + "bytes" + "io" + "os" + "testing" +) + +type zlibTest struct { + desc string + raw string + compressed []byte + err os.Error +} + +// Compare-to-golden test data was generated by the ZLIB example program at +// http://www.zlib.net/zpipe.c + +var zlibTests = []zlibTest{ + { + "empty", + "", + []byte{0x78, 0x9c, 0x03, 0x00, 0x00, 0x00, 0x00, 0x01}, + nil, + }, + { + "goodbye", + "goodbye, world", + []byte{ + 0x78, 0x9c, 0x4b, 0xcf, 0xcf, 0x4f, 0x49, 0xaa, + 0x4c, 0xd5, 0x51, 0x28, 0xcf, 0x2f, 0xca, 0x49, + 0x01, 0x00, 0x28, 0xa5, 0x05, 0x5e, + }, + nil, + }, + { + "bad header", + "", + []byte{0x78, 0x9f, 0x03, 0x00, 0x00, 0x00, 0x00, 0x01}, + HeaderError, + }, + { + "bad checksum", + "", + []byte{0x78, 0x9c, 0x03, 0x00, 0x00, 0x00, 0x00, 0xff}, + ChecksumError, + }, + { + "not enough data", + "", + []byte{0x78, 0x9c, 0x03, 0x00, 0x00, 0x00}, + io.ErrUnexpectedEOF, + }, + { + "excess data is silently ignored", + "", + []byte{ + 0x78, 0x9c, 0x03, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x78, 0x9c, 0xff, + }, + nil, + }, +} + +func TestDecompressor(t *testing.T) { + b := new(bytes.Buffer) + for _, tt := range zlibTests { + in := bytes.NewBuffer(tt.compressed) + zlib, err := NewReader(in) + if err != nil { + if err != tt.err { + t.Errorf("%s: NewReader: %s", tt.desc, err) + } + continue + } + defer zlib.Close() + b.Reset() + n, err := io.Copy(b, zlib) + if err != nil { + if err != tt.err { + t.Errorf("%s: io.Copy: %v want %v", tt.desc, err, tt.err) + } + continue + } + s := b.String() + if s != tt.raw { + t.Errorf("%s: got %d-byte %q want %d-byte %q", tt.desc, n, s, len(tt.raw), tt.raw) + } + } +} diff --git a/libgo/go/compress/zlib/testdata/e.txt b/libgo/go/compress/zlib/testdata/e.txt new file mode 100644 index 000000000..76cf2a7b6 --- /dev/null +++ b/libgo/go/compress/zlib/testdata/e.txt @@ -0,0 +1 @@ +2.7182818284590452353602874713526624977572470936999595749669676277240766303535475945713821785251664274274663919320030599218174135966290435729003342952605956307381323286279434907632338298807531952510190115738341879307021540891499348841675092447614606680822648001684774118537423454424371075390777449920695517027618386062613313845830007520449338265602976067371132007093287091274437470472306969772093101416928368190255151086574637721112523897844250569536967707854499699679468644549059879316368892300987931277361782154249992295763514822082698951936680331825288693984964651058209392398294887933203625094431173012381970684161403970198376793206832823764648042953118023287825098194558153017567173613320698112509961818815930416903515988885193458072738667385894228792284998920868058257492796104841984443634632449684875602336248270419786232090021609902353043699418491463140934317381436405462531520961836908887070167683964243781405927145635490613031072085103837505101157477041718986106873969655212671546889570350354021234078498193343210681701210056278802351930332247450158539047304199577770935036604169973297250886876966403555707162268447162560798826517871341951246652010305921236677194325278675398558944896970964097545918569563802363701621120477427228364896134225164450781824423529486363721417402388934412479635743702637552944483379980161254922785092577825620926226483262779333865664816277251640191059004916449982893150566047258027786318641551956532442586982946959308019152987211725563475463964479101459040905862984967912874068705048958586717479854667757573205681288459205413340539220001137863009455606881667400169842055804033637953764520304024322566135278369511778838638744396625322498506549958862342818997077332761717839280349465014345588970719425863987727547109629537415211151368350627526023264847287039207643100595841166120545297030236472549296669381151373227536450988890313602057248176585118063036442812314965507047510254465011727211555194866850800368532281831521960037356252794495158284188294787610852639813955990067376482922443752871846245780361929819713991475644882626039033814418232625150974827987779964373089970388867782271383605772978824125611907176639465070633045279546618550966661856647097113444740160704626215680717481877844371436988218559670959102596862002353718588748569652200050311734392073211390803293634479727355955277349071783793421637012050054513263835440001863239914907054797780566978533580489669062951194324730995876552368128590413832411607226029983305353708761389396391779574540161372236187893652605381558415871869255386061647798340254351284396129460352913325942794904337299085731580290958631382683291477116396337092400316894586360606458459251269946557248391865642097526850823075442545993769170419777800853627309417101634349076964237222943523661255725088147792231519747780605696725380171807763603462459278778465850656050780844211529697521890874019660906651803516501792504619501366585436632712549639908549144200014574760819302212066024330096412704894390397177195180699086998606636583232278709376502260149291011517177635944602023249300280401867723910288097866605651183260043688508817157238669842242201024950551881694803221002515426494639812873677658927688163598312477886520141174110913601164995076629077943646005851941998560162647907615321038727557126992518275687989302761761146162549356495903798045838182323368612016243736569846703785853305275833337939907521660692380533698879565137285593883499894707416181550125397064648171946708348197214488898790676503795903669672494992545279033729636162658976039498576741397359441023744329709355477982629614591442936451428617158587339746791897571211956187385783644758448423555581050025611492391518893099463428413936080383091662818811503715284967059741625628236092168075150177725387402564253470879089137291722828611515915683725241630772254406337875931059826760944203261924285317018781772960235413060672136046000389661093647095141417185777014180606443636815464440053316087783143174440811949422975599314011888683314832802706553833004693290115744147563139997221703804617092894579096271662260740718749975359212756084414737823303270330168237193648002173285734935947564334129943024850235732214597843282641421684878721673367010615094243456984401873312810107945127223737886126058165668053714396127888732527373890392890506865324138062796025930387727697783792868409325365880733988457218746021005311483351323850047827169376218004904795597959290591655470505777514308175112698985188408718564026035305583737832422924185625644255022672155980274012617971928047139600689163828665277009752767069777036439260224372841840883251848770472638440379530166905465937461619323840363893131364327137688841026811219891275223056256756254701725086349765367288605966752740868627407912856576996313789753034660616669804218267724560530660773899624218340859882071864682623215080288286359746839654358856685503773131296587975810501214916207656769950659715344763470320853215603674828608378656803073062657633469774295634643716709397193060876963495328846833613038829431040800296873869117066666146800015121143442256023874474325250769387077775193299942137277211258843608715834835626961661980572526612206797540621062080649882918454395301529982092503005498257043390553570168653120526495614857249257386206917403695213533732531666345466588597286659451136441370331393672118569553952108458407244323835586063106806964924851232632699514603596037297253198368423363904632136710116192821711150282801604488058802382031981493096369596735832742024988245684941273860566491352526706046234450549227581151709314921879592718001940968866986837037302200475314338181092708030017205935530520700706072233999463990571311587099635777359027196285061146514837526209565346713290025994397663114545902685898979115837093419370441155121920117164880566945938131183843765620627846310490346293950029458341164824114969758326011800731699437393506966295712410273239138741754923071862454543222039552735295240245903805744502892246886285336542213815722131163288112052146489805180092024719391710555390113943316681515828843687606961102505171007392762385553386272553538830960671644662370922646809671254061869502143176211668140097595281493907222601112681153108387317617323235263605838173151034595736538223534992935822836851007810884634349983518404451704270189381994243410090575376257767571118090088164183319201962623416288166521374717325477727783488774366518828752156685719506371936565390389449366421764003121527870222366463635755503565576948886549500270853923617105502131147413744106134445544192101336172996285694899193369184729478580729156088510396781959429833186480756083679551496636448965592948187851784038773326247051945050419847742014183947731202815886845707290544057510601285258056594703046836344592652552137008068752009593453607316226118728173928074623094685367823106097921599360019946237993434210687813497346959246469752506246958616909178573976595199392993995567542714654910456860702099012606818704984178079173924071945996323060254707901774527513186809982284730860766536866855516467702911336827563107223346726113705490795365834538637196235856312618387156774118738527722922594743373785695538456246801013905727871016512966636764451872465653730402443684140814488732957847348490003019477888020460324660842875351848364959195082888323206522128104190448047247949291342284951970022601310430062410717971502793433263407995960531446053230488528972917659876016667811937932372453857209607582277178483361613582612896226118129455927462767137794487586753657544861407611931125958512655759734573015333642630767985443385761715333462325270572005303988289499034259566232975782488735029259166825894456894655992658454762694528780516501720674785417887982276806536650641910973434528878338621726156269582654478205672987756426325321594294418039943217000090542650763095588465895171709147607437136893319469090981904501290307099566226620303182649365733698419555776963787624918852865686607600566025605445711337286840205574416030837052312242587223438854123179481388550075689381124935386318635287083799845692619981794523364087429591180747453419551420351726184200845509170845682368200897739455842679214273477560879644279202708312150156406341341617166448069815483764491573900121217041547872591998943825364950514771379399147205219529079396137621107238494290616357604596231253506068537651423115349665683715116604220796394466621163255157729070978473156278277598788136491951257483328793771571459091064841642678309949723674420175862269402159407924480541255360431317992696739157542419296607312393763542139230617876753958711436104089409966089471418340698362993675362621545247298464213752891079884381306095552622720837518629837066787224430195793793786072107254277289071732854874374355781966511716618330881129120245204048682200072344035025448202834254187884653602591506445271657700044521097735585897622655484941621714989532383421600114062950718490427789258552743035221396835679018076406042138307308774460170842688272261177180842664333651780002171903449234264266292261456004337383868335555343453004264818473989215627086095650629340405264943244261445665921291225648893569655009154306426134252668472594914314239398845432486327461842846655985332312210466259890141712103446084271616619001257195870793217569698544013397622096749454185407118446433946990162698351607848924514058940946395267807354579700307051163682519487701189764002827648414160587206184185297189154019688253289309149665345753571427318482016384644832499037886069008072709327673127581966563941148961716832980455139729506687604740915420428429993541025829113502241690769431668574242522509026939034814856451303069925199590436384028429267412573422447765584177886171737265462085498294498946787350929581652632072258992368768457017823038096567883112289305809140572610865884845873101658151167533327674887014829167419701512559782572707406431808601428149024146780472327597684269633935773542930186739439716388611764209004068663398856841681003872389214483176070116684503887212364367043314091155733280182977988736590916659612402021778558854876176161989370794380056663364884365089144805571039765214696027662583599051987042300179465536788 diff --git a/libgo/go/compress/zlib/testdata/pi.txt b/libgo/go/compress/zlib/testdata/pi.txt new file mode 100644 index 000000000..58d8f3b6d --- /dev/null +++ b/libgo/go/compress/zlib/testdata/pi.txt @@ -0,0 +1 @@ +3.1415926535897932384626433832795028841971693993751058209749445923078164062862089986280348253421170679821480865132823066470938446095505822317253594081284811174502841027019385211055596446229489549303819644288109756659334461284756482337867831652712019091456485669234603486104543266482133936072602491412737245870066063155881748815209209628292540917153643678925903600113305305488204665213841469519415116094330572703657595919530921861173819326117931051185480744623799627495673518857527248912279381830119491298336733624406566430860213949463952247371907021798609437027705392171762931767523846748184676694051320005681271452635608277857713427577896091736371787214684409012249534301465495853710507922796892589235420199561121290219608640344181598136297747713099605187072113499999983729780499510597317328160963185950244594553469083026425223082533446850352619311881710100031378387528865875332083814206171776691473035982534904287554687311595628638823537875937519577818577805321712268066130019278766111959092164201989380952572010654858632788659361533818279682303019520353018529689957736225994138912497217752834791315155748572424541506959508295331168617278558890750983817546374649393192550604009277016711390098488240128583616035637076601047101819429555961989467678374494482553797747268471040475346462080466842590694912933136770289891521047521620569660240580381501935112533824300355876402474964732639141992726042699227967823547816360093417216412199245863150302861829745557067498385054945885869269956909272107975093029553211653449872027559602364806654991198818347977535663698074265425278625518184175746728909777727938000816470600161452491921732172147723501414419735685481613611573525521334757418494684385233239073941433345477624168625189835694855620992192221842725502542568876717904946016534668049886272327917860857843838279679766814541009538837863609506800642251252051173929848960841284886269456042419652850222106611863067442786220391949450471237137869609563643719172874677646575739624138908658326459958133904780275900994657640789512694683983525957098258226205224894077267194782684826014769909026401363944374553050682034962524517493996514314298091906592509372216964615157098583874105978859597729754989301617539284681382686838689427741559918559252459539594310499725246808459872736446958486538367362226260991246080512438843904512441365497627807977156914359977001296160894416948685558484063534220722258284886481584560285060168427394522674676788952521385225499546667278239864565961163548862305774564980355936345681743241125150760694794510965960940252288797108931456691368672287489405601015033086179286809208747609178249385890097149096759852613655497818931297848216829989487226588048575640142704775551323796414515237462343645428584447952658678210511413547357395231134271661021359695362314429524849371871101457654035902799344037420073105785390621983874478084784896833214457138687519435064302184531910484810053706146806749192781911979399520614196634287544406437451237181921799983910159195618146751426912397489409071864942319615679452080951465502252316038819301420937621378559566389377870830390697920773467221825625996615014215030680384477345492026054146659252014974428507325186660021324340881907104863317346496514539057962685610055081066587969981635747363840525714591028970641401109712062804390397595156771577004203378699360072305587631763594218731251471205329281918261861258673215791984148488291644706095752706957220917567116722910981690915280173506712748583222871835209353965725121083579151369882091444210067510334671103141267111369908658516398315019701651511685171437657618351556508849099898599823873455283316355076479185358932261854896321329330898570642046752590709154814165498594616371802709819943099244889575712828905923233260972997120844335732654893823911932597463667305836041428138830320382490375898524374417029132765618093773444030707469211201913020330380197621101100449293215160842444859637669838952286847831235526582131449576857262433441893039686426243410773226978028073189154411010446823252716201052652272111660396665573092547110557853763466820653109896526918620564769312570586356620185581007293606598764861179104533488503461136576867532494416680396265797877185560845529654126654085306143444318586769751456614068007002378776591344017127494704205622305389945613140711270004078547332699390814546646458807972708266830634328587856983052358089330657574067954571637752542021149557615814002501262285941302164715509792592309907965473761255176567513575178296664547791745011299614890304639947132962107340437518957359614589019389713111790429782856475032031986915140287080859904801094121472213179476477726224142548545403321571853061422881375850430633217518297986622371721591607716692547487389866549494501146540628433663937900397692656721463853067360965712091807638327166416274888800786925602902284721040317211860820419000422966171196377921337575114959501566049631862947265473642523081770367515906735023507283540567040386743513622224771589150495309844489333096340878076932599397805419341447377441842631298608099888687413260472156951623965864573021631598193195167353812974167729478672422924654366800980676928238280689964004824354037014163149658979409243237896907069779422362508221688957383798623001593776471651228935786015881617557829735233446042815126272037343146531977774160319906655418763979293344195215413418994854447345673831624993419131814809277771038638773431772075456545322077709212019051660962804909263601975988281613323166636528619326686336062735676303544776280350450777235547105859548702790814356240145171806246436267945612753181340783303362542327839449753824372058353114771199260638133467768796959703098339130771098704085913374641442822772634659470474587847787201927715280731767907707157213444730605700733492436931138350493163128404251219256517980694113528013147013047816437885185290928545201165839341965621349143415956258658655705526904965209858033850722426482939728584783163057777560688876446248246857926039535277348030480290058760758251047470916439613626760449256274204208320856611906254543372131535958450687724602901618766795240616342522577195429162991930645537799140373404328752628889639958794757291746426357455254079091451357111369410911939325191076020825202618798531887705842972591677813149699009019211697173727847684726860849003377024242916513005005168323364350389517029893922334517220138128069650117844087451960121228599371623130171144484640903890644954440061986907548516026327505298349187407866808818338510228334508504860825039302133219715518430635455007668282949304137765527939751754613953984683393638304746119966538581538420568533862186725233402830871123282789212507712629463229563989898935821167456270102183564622013496715188190973038119800497340723961036854066431939509790190699639552453005450580685501956730229219139339185680344903982059551002263535361920419947455385938102343955449597783779023742161727111723643435439478221818528624085140066604433258885698670543154706965747458550332323342107301545940516553790686627333799585115625784322988273723198987571415957811196358330059408730681216028764962867446047746491599505497374256269010490377819868359381465741268049256487985561453723478673303904688383436346553794986419270563872931748723320837601123029911367938627089438799362016295154133714248928307220126901475466847653576164773794675200490757155527819653621323926406160136358155907422020203187277605277219005561484255518792530343513984425322341576233610642506390497500865627109535919465897514131034822769306247435363256916078154781811528436679570611086153315044521274739245449454236828860613408414863776700961207151249140430272538607648236341433462351897576645216413767969031495019108575984423919862916421939949072362346468441173940326591840443780513338945257423995082965912285085558215725031071257012668302402929525220118726767562204154205161841634847565169998116141010029960783869092916030288400269104140792886215078424516709087000699282120660418371806535567252532567532861291042487761825829765157959847035622262934860034158722980534989650226291748788202734209222245339856264766914905562842503912757710284027998066365825488926488025456610172967026640765590429099456815065265305371829412703369313785178609040708667114965583434347693385781711386455873678123014587687126603489139095620099393610310291616152881384379099042317473363948045759314931405297634757481193567091101377517210080315590248530906692037671922033229094334676851422144773793937517034436619910403375111735471918550464490263655128162288244625759163330391072253837421821408835086573917715096828874782656995995744906617583441375223970968340800535598491754173818839994469748676265516582765848358845314277568790029095170283529716344562129640435231176006651012412006597558512761785838292041974844236080071930457618932349229279650198751872127267507981255470958904556357921221033346697499235630254947802490114195212382815309114079073860251522742995818072471625916685451333123948049470791191532673430282441860414263639548000448002670496248201792896476697583183271314251702969234889627668440323260927524960357996469256504936818360900323809293459588970695365349406034021665443755890045632882250545255640564482465151875471196218443965825337543885690941130315095261793780029741207665147939425902989695946995565761218656196733786236256125216320862869222103274889218654364802296780705765615144632046927906821207388377814233562823608963208068222468012248261177185896381409183903673672220888321513755600372798394004152970028783076670944474560134556417254370906979396122571429894671543578468788614445812314593571984922528471605049221242470141214780573455105008019086996033027634787081081754501193071412233908663938339529425786905076431006383519834389341596131854347546495569781038293097164651438407007073604112373599843452251610507027056235266012764848308407611830130527932054274628654036036745328651057065874882256981579367897669742205750596834408697350201410206723585020072452256326513410559240190274216248439140359989535394590944070469120914093870012645600162374288021092764579310657922955249887275846101264836999892256959688159205600101655256375678 diff --git a/libgo/go/compress/zlib/writer.go b/libgo/go/compress/zlib/writer.go new file mode 100644 index 000000000..031586cd2 --- /dev/null +++ b/libgo/go/compress/zlib/writer.go @@ -0,0 +1,106 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package zlib + +import ( + "compress/flate" + "hash" + "hash/adler32" + "io" + "os" +) + +// These constants are copied from the flate package, so that code that imports +// "compress/zlib" does not also have to import "compress/flate". +const ( + NoCompression = flate.NoCompression + BestSpeed = flate.BestSpeed + BestCompression = flate.BestCompression + DefaultCompression = flate.DefaultCompression +) + +type writer struct { + w io.Writer + compressor io.WriteCloser + digest hash.Hash32 + err os.Error + scratch [4]byte +} + +// NewWriter calls NewWriterLevel with the default compression level. +func NewWriter(w io.Writer) (io.WriteCloser, os.Error) { + return NewWriterLevel(w, DefaultCompression) +} + +// NewWriterLevel creates a new io.WriteCloser that satisfies writes by compressing data written to w. +// It is the caller's responsibility to call Close on the WriteCloser when done. +// level is the compression level, which can be DefaultCompression, NoCompression, +// or any integer value between BestSpeed and BestCompression (inclusive). +func NewWriterLevel(w io.Writer, level int) (io.WriteCloser, os.Error) { + z := new(writer) + // ZLIB has a two-byte header (as documented in RFC 1950). + // The first four bits is the CINFO (compression info), which is 7 for the default deflate window size. + // The next four bits is the CM (compression method), which is 8 for deflate. + z.scratch[0] = 0x78 + // The next two bits is the FLEVEL (compression level). The four values are: + // 0=fastest, 1=fast, 2=default, 3=best. + // The next bit, FDICT, is unused, in this implementation. + // The final five FCHECK bits form a mod-31 checksum. + switch level { + case 0, 1: + z.scratch[1] = 0x01 + case 2, 3, 4, 5: + z.scratch[1] = 0x5e + case 6, -1: + z.scratch[1] = 0x9c + case 7, 8, 9: + z.scratch[1] = 0xda + default: + return nil, os.NewError("level out of range") + } + _, err := w.Write(z.scratch[0:2]) + if err != nil { + return nil, err + } + z.w = w + z.compressor = flate.NewWriter(w, level) + z.digest = adler32.New() + return z, nil +} + +func (z *writer) Write(p []byte) (n int, err os.Error) { + if z.err != nil { + return 0, z.err + } + if len(p) == 0 { + return 0, nil + } + n, err = z.compressor.Write(p) + if err != nil { + z.err = err + return + } + z.digest.Write(p) + return +} + +// Calling Close does not close the wrapped io.Writer originally passed to NewWriter. +func (z *writer) Close() os.Error { + if z.err != nil { + return z.err + } + z.err = z.compressor.Close() + if z.err != nil { + return z.err + } + checksum := z.digest.Sum32() + // ZLIB (RFC 1950) is big-endian, unlike GZIP (RFC 1952). + z.scratch[0] = uint8(checksum >> 24) + z.scratch[1] = uint8(checksum >> 16) + z.scratch[2] = uint8(checksum >> 8) + z.scratch[3] = uint8(checksum >> 0) + _, z.err = z.w.Write(z.scratch[0:4]) + return z.err +} diff --git a/libgo/go/compress/zlib/writer_test.go b/libgo/go/compress/zlib/writer_test.go new file mode 100644 index 000000000..fa9e78e8e --- /dev/null +++ b/libgo/go/compress/zlib/writer_test.go @@ -0,0 +1,106 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package zlib + +import ( + "io" + "io/ioutil" + "os" + "testing" +) + +var filenames = []string{ + "testdata/e.txt", + "testdata/pi.txt", +} + +// Tests that compressing and then decompressing the given file at the given compression level +// yields equivalent bytes to the original file. +func testFileLevel(t *testing.T, fn string, level int) { + // Read the file, as golden output. + golden, err := os.Open(fn, os.O_RDONLY, 0444) + if err != nil { + t.Errorf("%s (level=%d): %v", fn, level, err) + return + } + defer golden.Close() + + // Read the file again, and push it through a pipe that compresses at the write end, and decompresses at the read end. + raw, err := os.Open(fn, os.O_RDONLY, 0444) + if err != nil { + t.Errorf("%s (level=%d): %v", fn, level, err) + return + } + piper, pipew := io.Pipe() + defer piper.Close() + go func() { + defer raw.Close() + defer pipew.Close() + zlibw, err := NewWriterLevel(pipew, level) + if err != nil { + t.Errorf("%s (level=%d): %v", fn, level, err) + return + } + defer zlibw.Close() + var b [1024]byte + for { + n, err0 := raw.Read(b[0:]) + if err0 != nil && err0 != os.EOF { + t.Errorf("%s (level=%d): %v", fn, level, err0) + return + } + _, err1 := zlibw.Write(b[0:n]) + if err1 == os.EPIPE { + // Fail, but do not report the error, as some other (presumably reportable) error broke the pipe. + return + } + if err1 != nil { + t.Errorf("%s (level=%d): %v", fn, level, err1) + return + } + if err0 == os.EOF { + break + } + } + }() + zlibr, err := NewReader(piper) + if err != nil { + t.Errorf("%s (level=%d): %v", fn, level, err) + return + } + defer zlibr.Close() + + // Compare the two. + b0, err0 := ioutil.ReadAll(golden) + b1, err1 := ioutil.ReadAll(zlibr) + if err0 != nil { + t.Errorf("%s (level=%d): %v", fn, level, err0) + return + } + if err1 != nil { + t.Errorf("%s (level=%d): %v", fn, level, err1) + return + } + if len(b0) != len(b1) { + t.Errorf("%s (level=%d): length mismatch %d versus %d", fn, level, len(b0), len(b1)) + return + } + for i := 0; i < len(b0); i++ { + if b0[i] != b1[i] { + t.Errorf("%s (level=%d): mismatch at %d, 0x%02x versus 0x%02x\n", fn, level, i, b0[i], b1[i]) + return + } + } +} + +func TestWriter(t *testing.T) { + for _, fn := range filenames { + testFileLevel(t, fn, DefaultCompression) + testFileLevel(t, fn, NoCompression) + for level := BestSpeed; level <= BestCompression; level++ { + testFileLevel(t, fn, level) + } + } +} diff --git a/libgo/go/container/heap/heap.go b/libgo/go/container/heap/heap.go new file mode 100644 index 000000000..4435a57c4 --- /dev/null +++ b/libgo/go/container/heap/heap.go @@ -0,0 +1,102 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This package provides heap operations for any type that implements +// heap.Interface. +// +package heap + +import "sort" + +// Any type that implements heap.Interface may be used as a +// min-heap with the following invariants (established after +// Init has been called): +// +// !h.Less(j, i) for 0 <= i < h.Len() and j = 2*i+1 or 2*i+2 and j < h.Len() +// +type Interface interface { + sort.Interface + Push(x interface{}) + Pop() interface{} +} + + +// A heaper must be initialized before any of the heap operations +// can be used. Init is idempotent with respect to the heap invariants +// and may be called whenever the heap invariants may have been invalidated. +// Its complexity is O(n) where n = h.Len(). +// +func Init(h Interface) { + // heapify + n := h.Len() + for i := n/2 - 1; i >= 0; i-- { + down(h, i, n) + } +} + + +// Push pushes the element x onto the heap. The complexity is +// O(log(n)) where n = h.Len(). +// +func Push(h Interface, x interface{}) { + h.Push(x) + up(h, h.Len()-1) +} + + +// Pop removes the minimum element (according to Less) from the heap +// and returns it. The complexity is O(log(n)) where n = h.Len(). +// Same as Remove(h, 0). +// +func Pop(h Interface) interface{} { + n := h.Len() - 1 + h.Swap(0, n) + down(h, 0, n) + return h.Pop() +} + + +// Remove removes the element at index i from the heap. +// The complexity is O(log(n)) where n = h.Len(). +// +func Remove(h Interface, i int) interface{} { + n := h.Len() - 1 + if n != i { + h.Swap(i, n) + down(h, i, n) + up(h, i) + } + return h.Pop() +} + + +func up(h Interface, j int) { + for { + i := (j - 1) / 2 // parent + if i == j || h.Less(i, j) { + break + } + h.Swap(i, j) + j = i + } +} + + +func down(h Interface, i, n int) { + for { + j1 := 2*i + 1 + if j1 >= n { + break + } + j := j1 // left child + if j2 := j1 + 1; j2 < n && !h.Less(j1, j2) { + j = j2 // = 2*i + 2 // right child + } + if h.Less(i, j) { + break + } + h.Swap(i, j) + i = j + } +} diff --git a/libgo/go/container/heap/heap_test.go b/libgo/go/container/heap/heap_test.go new file mode 100644 index 000000000..89d444dd5 --- /dev/null +++ b/libgo/go/container/heap/heap_test.go @@ -0,0 +1,166 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package heap + +import ( + "testing" + "container/vector" +) + + +type myHeap struct { + // A vector.Vector implements sort.Interface except for Less, + // and it implements Push and Pop as required for heap.Interface. + vector.Vector +} + + +func (h *myHeap) Less(i, j int) bool { return h.At(i).(int) < h.At(j).(int) } + + +func (h *myHeap) verify(t *testing.T, i int) { + n := h.Len() + j1 := 2*i + 1 + j2 := 2*i + 2 + if j1 < n { + if h.Less(j1, i) { + t.Errorf("heap invariant invalidated [%d] = %d > [%d] = %d", i, h.At(i), j1, h.At(j1)) + return + } + h.verify(t, j1) + } + if j2 < n { + if h.Less(j2, i) { + t.Errorf("heap invariant invalidated [%d] = %d > [%d] = %d", i, h.At(i), j1, h.At(j2)) + return + } + h.verify(t, j2) + } +} + + +func TestInit0(t *testing.T) { + h := new(myHeap) + for i := 20; i > 0; i-- { + h.Push(0) // all elements are the same + } + Init(h) + h.verify(t, 0) + + for i := 1; h.Len() > 0; i++ { + x := Pop(h).(int) + h.verify(t, 0) + if x != 0 { + t.Errorf("%d.th pop got %d; want %d", i, x, 0) + } + } +} + + +func TestInit1(t *testing.T) { + h := new(myHeap) + for i := 20; i > 0; i-- { + h.Push(i) // all elements are different + } + Init(h) + h.verify(t, 0) + + for i := 1; h.Len() > 0; i++ { + x := Pop(h).(int) + h.verify(t, 0) + if x != i { + t.Errorf("%d.th pop got %d; want %d", i, x, i) + } + } +} + + +func Test(t *testing.T) { + h := new(myHeap) + h.verify(t, 0) + + for i := 20; i > 10; i-- { + h.Push(i) + } + Init(h) + h.verify(t, 0) + + for i := 10; i > 0; i-- { + Push(h, i) + h.verify(t, 0) + } + + for i := 1; h.Len() > 0; i++ { + x := Pop(h).(int) + if i < 20 { + Push(h, 20+i) + } + h.verify(t, 0) + if x != i { + t.Errorf("%d.th pop got %d; want %d", i, x, i) + } + } +} + + +func TestRemove0(t *testing.T) { + h := new(myHeap) + for i := 0; i < 10; i++ { + h.Push(i) + } + h.verify(t, 0) + + for h.Len() > 0 { + i := h.Len() - 1 + x := Remove(h, i).(int) + if x != i { + t.Errorf("Remove(%d) got %d; want %d", i, x, i) + } + h.verify(t, 0) + } +} + + +func TestRemove1(t *testing.T) { + h := new(myHeap) + for i := 0; i < 10; i++ { + h.Push(i) + } + h.verify(t, 0) + + for i := 0; h.Len() > 0; i++ { + x := Remove(h, 0).(int) + if x != i { + t.Errorf("Remove(0) got %d; want %d", x, i) + } + h.verify(t, 0) + } +} + + +func TestRemove2(t *testing.T) { + N := 10 + + h := new(myHeap) + for i := 0; i < N; i++ { + h.Push(i) + } + h.verify(t, 0) + + m := make(map[int]bool) + for h.Len() > 0 { + m[Remove(h, (h.Len()-1)/2).(int)] = true + h.verify(t, 0) + } + + if len(m) != N { + t.Errorf("len(m) = %d; want %d", len(m), N) + } + for i := 0; i < len(m); i++ { + if !m[i] { + t.Errorf("m[%d] doesn't exist", i) + } + } +} diff --git a/libgo/go/container/list/list.go b/libgo/go/container/list/list.go new file mode 100755 index 000000000..c1ebcddaa --- /dev/null +++ b/libgo/go/container/list/list.go @@ -0,0 +1,211 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The list package implements a doubly linked list. +// +// To iterate over a list (where l is a *List): +// for e := l.Front(); e != nil; e = e.Next() { +// // do something with e.Value +// } +// +package list + +// Element is an element in the linked list. +type Element struct { + // Next and previous pointers in the doubly-linked list of elements. + // The front of the list has prev = nil, and the back has next = nil. + next, prev *Element + + // The list to which this element belongs. + list *List + + // The contents of this list element. + Value interface{} +} + +// Next returns the next list element or nil. +func (e *Element) Next() *Element { return e.next } + +// Prev returns the previous list element or nil. +func (e *Element) Prev() *Element { return e.prev } + +// List represents a doubly linked list. +// The zero value for List is an empty list ready to use. +type List struct { + front, back *Element + len int +} + +// Init initializes or clears a List. +func (l *List) Init() *List { + l.front = nil + l.back = nil + l.len = 0 + return l +} + +// New returns an initialized list. +func New() *List { return new(List) } + +// Front returns the first element in the list. +func (l *List) Front() *Element { return l.front } + +// Back returns the last element in the list. +func (l *List) Back() *Element { return l.back } + +// Remove removes the element from the list +// and returns its Value. +func (l *List) Remove(e *Element) interface{} { + l.remove(e) + e.list = nil // do what remove does not + return e.Value +} + +// remove the element from the list, but do not clear the Element's list field. +// This is so that other List methods may use remove when relocating Elements +// without needing to restore the list field. +func (l *List) remove(e *Element) { + if e.list != l { + return + } + if e.prev == nil { + l.front = e.next + } else { + e.prev.next = e.next + } + if e.next == nil { + l.back = e.prev + } else { + e.next.prev = e.prev + } + + e.prev = nil + e.next = nil + l.len-- +} + +func (l *List) insertBefore(e *Element, mark *Element) { + if mark.prev == nil { + // new front of the list + l.front = e + } else { + mark.prev.next = e + } + e.prev = mark.prev + mark.prev = e + e.next = mark + l.len++ +} + +func (l *List) insertAfter(e *Element, mark *Element) { + if mark.next == nil { + // new back of the list + l.back = e + } else { + mark.next.prev = e + } + e.next = mark.next + mark.next = e + e.prev = mark + l.len++ +} + +func (l *List) insertFront(e *Element) { + if l.front == nil { + // empty list + l.front, l.back = e, e + e.prev, e.next = nil, nil + l.len = 1 + return + } + l.insertBefore(e, l.front) +} + +func (l *List) insertBack(e *Element) { + if l.back == nil { + // empty list + l.front, l.back = e, e + e.prev, e.next = nil, nil + l.len = 1 + return + } + l.insertAfter(e, l.back) +} + +// PushFront inserts the value at the front of the list and returns a new Element containing the value. +func (l *List) PushFront(value interface{}) *Element { + e := &Element{nil, nil, l, value} + l.insertFront(e) + return e +} + +// PushBack inserts the value at the back of the list and returns a new Element containing the value. +func (l *List) PushBack(value interface{}) *Element { + e := &Element{nil, nil, l, value} + l.insertBack(e) + return e +} + +// InsertBefore inserts the value immediately before mark and returns a new Element containing the value. +func (l *List) InsertBefore(value interface{}, mark *Element) *Element { + if mark.list != l { + return nil + } + e := &Element{nil, nil, l, value} + l.insertBefore(e, mark) + return e +} + +// InsertAfter inserts the value immediately after mark and returns a new Element containing the value. +func (l *List) InsertAfter(value interface{}, mark *Element) *Element { + if mark.list != l { + return nil + } + e := &Element{nil, nil, l, value} + l.insertAfter(e, mark) + return e +} + +// MoveToFront moves the element to the front of the list. +func (l *List) MoveToFront(e *Element) { + if e.list != l || l.front == e { + return + } + l.remove(e) + l.insertFront(e) +} + +// MoveToBack moves the element to the back of the list. +func (l *List) MoveToBack(e *Element) { + if e.list != l || l.back == e { + return + } + l.remove(e) + l.insertBack(e) +} + +// Len returns the number of elements in the list. +func (l *List) Len() int { return l.len } + +// PushBackList inserts each element of ol at the back of the list. +func (l *List) PushBackList(ol *List) { + last := ol.Back() + for e := ol.Front(); e != nil; e = e.Next() { + l.PushBack(e.Value) + if e == last { + break + } + } +} + +// PushFrontList inserts each element of ol at the front of the list. The ordering of the passed list is preserved. +func (l *List) PushFrontList(ol *List) { + first := ol.Front() + for e := ol.Back(); e != nil; e = e.Prev() { + l.PushFront(e.Value) + if e == first { + break + } + } +} diff --git a/libgo/go/container/list/list_test.go b/libgo/go/container/list/list_test.go new file mode 100755 index 000000000..1d44ff84e --- /dev/null +++ b/libgo/go/container/list/list_test.go @@ -0,0 +1,209 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package list + +import ( + "testing" +) + +func checkListPointers(t *testing.T, l *List, es []*Element) { + if len(es) == 0 { + if l.front != nil || l.back != nil { + t.Errorf("l.front/l.back = %v/%v should be nil/nil", l.front, l.back) + } + return + } + + if l.front != es[0] { + t.Errorf("l.front = %v, want %v", l.front, es[0]) + } + if last := es[len(es)-1]; l.back != last { + t.Errorf("l.back = %v, want %v", l.back, last) + } + + for i, e := range es { + var e_prev, e_next *Element = nil, nil + if i > 0 { + e_prev = es[i-1] + } + if i < len(es)-1 { + e_next = es[i+1] + } + if e.prev != e_prev { + t.Errorf("elt #%d (%v) has prev=%v, want %v", i, e, e.prev, e_prev) + } + if e.next != e_next { + t.Errorf("elt #%d (%v) has next=%v, want %v", i, e, e.next, e_next) + } + } +} + +func checkListLen(t *testing.T, l *List, n int) { + if an := l.Len(); an != n { + t.Errorf("l.Len() = %d, want %d", an, n) + } +} + +func TestList(t *testing.T) { + l := New() + checkListPointers(t, l, []*Element{}) + checkListLen(t, l, 0) + + // Single element list + e := l.PushFront("a") + checkListLen(t, l, 1) + checkListPointers(t, l, []*Element{e}) + l.MoveToFront(e) + checkListPointers(t, l, []*Element{e}) + l.MoveToBack(e) + checkListPointers(t, l, []*Element{e}) + checkListLen(t, l, 1) + l.Remove(e) + checkListPointers(t, l, []*Element{}) + checkListLen(t, l, 0) + + // Bigger list + e2 := l.PushFront(2) + e1 := l.PushFront(1) + e3 := l.PushBack(3) + e4 := l.PushBack("banana") + checkListPointers(t, l, []*Element{e1, e2, e3, e4}) + checkListLen(t, l, 4) + + l.Remove(e2) + checkListPointers(t, l, []*Element{e1, e3, e4}) + checkListLen(t, l, 3) + + l.MoveToFront(e3) // move from middle + checkListPointers(t, l, []*Element{e3, e1, e4}) + + l.MoveToFront(e1) + l.MoveToBack(e3) // move from middle + checkListPointers(t, l, []*Element{e1, e4, e3}) + + l.MoveToFront(e3) // move from back + checkListPointers(t, l, []*Element{e3, e1, e4}) + l.MoveToFront(e3) // should be no-op + checkListPointers(t, l, []*Element{e3, e1, e4}) + + l.MoveToBack(e3) // move from front + checkListPointers(t, l, []*Element{e1, e4, e3}) + l.MoveToBack(e3) // should be no-op + checkListPointers(t, l, []*Element{e1, e4, e3}) + + e2 = l.InsertBefore(2, e1) // insert before front + checkListPointers(t, l, []*Element{e2, e1, e4, e3}) + l.Remove(e2) + e2 = l.InsertBefore(2, e4) // insert before middle + checkListPointers(t, l, []*Element{e1, e2, e4, e3}) + l.Remove(e2) + e2 = l.InsertBefore(2, e3) // insert before back + checkListPointers(t, l, []*Element{e1, e4, e2, e3}) + l.Remove(e2) + + e2 = l.InsertAfter(2, e1) // insert after front + checkListPointers(t, l, []*Element{e1, e2, e4, e3}) + l.Remove(e2) + e2 = l.InsertAfter(2, e4) // insert after middle + checkListPointers(t, l, []*Element{e1, e4, e2, e3}) + l.Remove(e2) + e2 = l.InsertAfter(2, e3) // insert after back + checkListPointers(t, l, []*Element{e1, e4, e3, e2}) + l.Remove(e2) + + // Check standard iteration. + sum := 0 + for e := l.Front(); e != nil; e = e.Next() { + if i, ok := e.Value.(int); ok { + sum += i + } + } + if sum != 4 { + t.Errorf("sum over l.Iter() = %d, want 4", sum) + } + + // Clear all elements by iterating + var next *Element + for e := l.Front(); e != nil; e = next { + next = e.Next() + l.Remove(e) + } + checkListPointers(t, l, []*Element{}) + checkListLen(t, l, 0) +} + +func checkList(t *testing.T, l *List, es []interface{}) { + if l.Len() != len(es) { + t.Errorf("list has len=%v, want %v", l.Len(), len(es)) + return + } + i := 0 + for e := l.Front(); e != nil; e = e.Next() { + le := e.Value.(int) + if le != es[i] { + t.Errorf("elt #%d has value=%v, want %v", i, le, es[i]) + } + i++ + } +} + +func TestExtending(t *testing.T) { + l1 := New() + l2 := New() + + l1.PushBack(1) + l1.PushBack(2) + l1.PushBack(3) + + l2.PushBack(4) + l2.PushBack(5) + + l3 := New() + l3.PushBackList(l1) + checkList(t, l3, []interface{}{1, 2, 3}) + l3.PushBackList(l2) + checkList(t, l3, []interface{}{1, 2, 3, 4, 5}) + + l3 = New() + l3.PushFrontList(l2) + checkList(t, l3, []interface{}{4, 5}) + l3.PushFrontList(l1) + checkList(t, l3, []interface{}{1, 2, 3, 4, 5}) + + checkList(t, l1, []interface{}{1, 2, 3}) + checkList(t, l2, []interface{}{4, 5}) + + l3 = New() + l3.PushBackList(l1) + checkList(t, l3, []interface{}{1, 2, 3}) + l3.PushBackList(l3) + checkList(t, l3, []interface{}{1, 2, 3, 1, 2, 3}) + + l3 = New() + l3.PushFrontList(l1) + checkList(t, l3, []interface{}{1, 2, 3}) + l3.PushFrontList(l3) + checkList(t, l3, []interface{}{1, 2, 3, 1, 2, 3}) + + l3 = New() + l1.PushBackList(l3) + checkList(t, l1, []interface{}{1, 2, 3}) + l1.PushFrontList(l3) + checkList(t, l1, []interface{}{1, 2, 3}) +} + +func TestRemove(t *testing.T) { + l := New() + e1 := l.PushBack(1) + e2 := l.PushBack(2) + checkListPointers(t, l, []*Element{e1, e2}) + e := l.Front() + l.Remove(e) + checkListPointers(t, l, []*Element{e2}) + checkListLen(t, l, 1) + l.Remove(e) + checkListPointers(t, l, []*Element{e2}) + checkListLen(t, l, 1) +} diff --git a/libgo/go/container/ring/ring.go b/libgo/go/container/ring/ring.go new file mode 100644 index 000000000..335afbc3c --- /dev/null +++ b/libgo/go/container/ring/ring.go @@ -0,0 +1,153 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The ring package implements operations on circular lists. +package ring + +// A Ring is an element of a circular list, or ring. +// Rings do not have a beginning or end; a pointer to any ring element +// serves as reference to the entire ring. Empty rings are represented +// as nil Ring pointers. The zero value for a Ring is a one-element +// ring with a nil Value. +// +type Ring struct { + next, prev *Ring + Value interface{} // for use by client; untouched by this library +} + + +func (r *Ring) init() *Ring { + r.next = r + r.prev = r + return r +} + + +// Next returns the next ring element. r must not be empty. +func (r *Ring) Next() *Ring { + if r.next == nil { + return r.init() + } + return r.next +} + + +// Prev returns the previous ring element. r must not be empty. +func (r *Ring) Prev() *Ring { + if r.next == nil { + return r.init() + } + return r.prev +} + + +// Move moves n % r.Len() elements backward (n < 0) or forward (n >= 0) +// in the ring and returns that ring element. r must not be empty. +// +func (r *Ring) Move(n int) *Ring { + if r.next == nil { + return r.init() + } + switch { + case n < 0: + for ; n < 0; n++ { + r = r.prev + } + case n > 0: + for ; n > 0; n-- { + r = r.next + } + } + return r +} + + +// New creates a ring of n elements. +func New(n int) *Ring { + if n <= 0 { + return nil + } + r := new(Ring) + p := r + for i := 1; i < n; i++ { + p.next = &Ring{prev: p} + p = p.next + } + p.next = r + r.prev = p + return r +} + + +// Link connects ring r with with ring s such that r.Next() +// becomes s and returns the original value for r.Next(). +// r must not be empty. +// +// If r and s point to the same ring, linking +// them removes the elements between r and s from the ring. +// The removed elements form a subring and the result is a +// reference to that subring (if no elements were removed, +// the result is still the original value for r.Next(), +// and not nil). +// +// If r and s point to different rings, linking +// them creates a single ring with the elements of s inserted +// after r. The result points to the element following the +// last element of s after insertion. +// +func (r *Ring) Link(s *Ring) *Ring { + n := r.Next() + if s != nil { + p := s.Prev() + // Note: Cannot use multiple assignment because + // evaluation order of LHS is not specified. + r.next = s + s.prev = r + n.prev = p + p.next = n + } + return n +} + + +// Unlink removes n % r.Len() elements from the ring r, starting +// at r.Next(). If n % r.Len() == 0, r remains unchanged. +// The result is the removed subring. r must not be empty. +// +func (r *Ring) Unlink(n int) *Ring { + if n <= 0 { + return nil + } + return r.Link(r.Move(n + 1)) +} + + +// Len computes the number of elements in ring r. +// It executes in time proportional to the number of elements. +// +func (r *Ring) Len() int { + n := 0 + if r != nil { + n = 1 + for p := r.Next(); p != r; p = p.next { + n++ + } + } + return n +} + + +func (r *Ring) Iter() <-chan interface{} { + c := make(chan interface{}) + go func() { + if r != nil { + c <- r.Value + for p := r.Next(); p != r; p = p.next { + c <- p.Value + } + } + close(c) + }() + return c +} diff --git a/libgo/go/container/ring/ring_test.go b/libgo/go/container/ring/ring_test.go new file mode 100644 index 000000000..ee3c41128 --- /dev/null +++ b/libgo/go/container/ring/ring_test.go @@ -0,0 +1,240 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ring + +import ( + "fmt" + "testing" +) + + +// For debugging - keep around. +func dump(r *Ring) { + if r == nil { + fmt.Println("empty") + return + } + i, n := 0, r.Len() + for p := r; i < n; p = p.next { + fmt.Printf("%4d: %p = {<- %p | %p ->}\n", i, p, p.prev, p.next) + i++ + } + fmt.Println() +} + + +func verify(t *testing.T, r *Ring, N int, sum int) { + // Len + n := r.Len() + if n != N { + t.Errorf("r.Len() == %d; expected %d", n, N) + } + + // iteration + n = 0 + s := 0 + for p := range r.Iter() { + n++ + if p != nil { + s += p.(int) + } + } + if n != N { + t.Errorf("number of forward iterations == %d; expected %d", n, N) + } + if sum >= 0 && s != sum { + t.Errorf("forward ring sum = %d; expected %d", s, sum) + } + + if r == nil { + return + } + + // connections + if r.next != nil { + var p *Ring // previous element + for q := r; p == nil || q != r; q = q.next { + if p != nil && p != q.prev { + t.Errorf("prev = %p, expected q.prev = %p\n", p, q.prev) + } + p = q + } + if p != r.prev { + t.Errorf("prev = %p, expected r.prev = %p\n", p, r.prev) + } + } + + // Next, Prev + if r.Next() != r.next { + t.Errorf("r.Next() != r.next") + } + if r.Prev() != r.prev { + t.Errorf("r.Prev() != r.prev") + } + + // Move + if r.Move(0) != r { + t.Errorf("r.Move(0) != r") + } + if r.Move(N) != r { + t.Errorf("r.Move(%d) != r", N) + } + if r.Move(-N) != r { + t.Errorf("r.Move(%d) != r", -N) + } + for i := 0; i < 10; i++ { + ni := N + i + mi := ni % N + if r.Move(ni) != r.Move(mi) { + t.Errorf("r.Move(%d) != r.Move(%d)", ni, mi) + } + if r.Move(-ni) != r.Move(-mi) { + t.Errorf("r.Move(%d) != r.Move(%d)", -ni, -mi) + } + } +} + + +func TestCornerCases(t *testing.T) { + var ( + r0 *Ring + r1 Ring + ) + // Basics + verify(t, r0, 0, 0) + verify(t, &r1, 1, 0) + // Insert + r1.Link(r0) + verify(t, r0, 0, 0) + verify(t, &r1, 1, 0) + // Insert + r1.Link(r0) + verify(t, r0, 0, 0) + verify(t, &r1, 1, 0) + // Unlink + r1.Unlink(0) + verify(t, &r1, 1, 0) +} + + +func makeN(n int) *Ring { + r := New(n) + for i := 1; i <= n; i++ { + r.Value = i + r = r.Next() + } + return r +} + + +func sum(r *Ring) int { + s := 0 + for p := range r.Iter() { + s += p.(int) + } + return s +} + + +func sumN(n int) int { return (n*n + n) / 2 } + + +func TestNew(t *testing.T) { + for i := 0; i < 10; i++ { + r := New(i) + verify(t, r, i, -1) + } + for i := 0; i < 10; i++ { + r := makeN(i) + verify(t, r, i, sumN(i)) + } +} + + +func TestLink1(t *testing.T) { + r1a := makeN(1) + var r1b Ring + r2a := r1a.Link(&r1b) + verify(t, r2a, 2, 1) + if r2a != r1a { + t.Errorf("a) 2-element link failed") + } + + r2b := r2a.Link(r2a.Next()) + verify(t, r2b, 2, 1) + if r2b != r2a.Next() { + t.Errorf("b) 2-element link failed") + } + + r1c := r2b.Link(r2b) + verify(t, r1c, 1, 1) + verify(t, r2b, 1, 0) +} + + +func TestLink2(t *testing.T) { + var r0 *Ring + r1a := &Ring{Value: 42} + r1b := &Ring{Value: 77} + r10 := makeN(10) + + r1a.Link(r0) + verify(t, r1a, 1, 42) + + r1a.Link(r1b) + verify(t, r1a, 2, 42+77) + + r10.Link(r0) + verify(t, r10, 10, sumN(10)) + + r10.Link(r1a) + verify(t, r10, 12, sumN(10)+42+77) +} + + +func TestLink3(t *testing.T) { + var r Ring + n := 1 + for i := 1; i < 100; i++ { + n += i + verify(t, r.Link(New(i)), n, -1) + } +} + + +func TestUnlink(t *testing.T) { + r10 := makeN(10) + s10 := r10.Move(6) + + sum10 := sumN(10) + + verify(t, r10, 10, sum10) + verify(t, s10, 10, sum10) + + r0 := r10.Unlink(0) + verify(t, r0, 0, 0) + + r1 := r10.Unlink(1) + verify(t, r1, 1, 2) + verify(t, r10, 9, sum10-2) + + r9 := r10.Unlink(9) + verify(t, r9, 9, sum10-2) + verify(t, r10, 9, sum10-2) +} + + +func TestLinkUnlink(t *testing.T) { + for i := 1; i < 4; i++ { + ri := New(i) + for j := 0; j < i; j++ { + rj := ri.Unlink(j) + verify(t, rj, j, -1) + verify(t, ri, i-j, -1) + ri.Link(rj) + verify(t, ri, i, -1) + } + } +} diff --git a/libgo/go/container/vector/defs.go b/libgo/go/container/vector/defs.go new file mode 100644 index 000000000..a2febb6de --- /dev/null +++ b/libgo/go/container/vector/defs.go @@ -0,0 +1,51 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The vector package implements containers for managing sequences +// of elements. Vectors grow and shrink dynamically as necessary. +package vector + + +// Vector is a container for numbered sequences of elements of type interface{}. +// A vector's length and capacity adjusts automatically as necessary. +// The zero value for Vector is an empty vector ready to use. +type Vector []interface{} + + +// IntVector is a container for numbered sequences of elements of type int. +// A vector's length and capacity adjusts automatically as necessary. +// The zero value for IntVector is an empty vector ready to use. +type IntVector []int + + +// StringVector is a container for numbered sequences of elements of type string. +// A vector's length and capacity adjusts automatically as necessary. +// The zero value for StringVector is an empty vector ready to use. +type StringVector []string + + +// Initial underlying array size +const initialSize = 8 + + +// Partial sort.Interface support + +// LessInterface provides partial support of the sort.Interface. +type LessInterface interface { + Less(y interface{}) bool +} + + +// Less returns a boolean denoting whether the i'th element is less than the j'th element. +func (p *Vector) Less(i, j int) bool { return (*p)[i].(LessInterface).Less((*p)[j]) } + + +// sort.Interface support + +// Less returns a boolean denoting whether the i'th element is less than the j'th element. +func (p *IntVector) Less(i, j int) bool { return (*p)[i] < (*p)[j] } + + +// Less returns a boolean denoting whether the i'th element is less than the j'th element. +func (p *StringVector) Less(i, j int) bool { return (*p)[i] < (*p)[j] } diff --git a/libgo/go/container/vector/intvector.go b/libgo/go/container/vector/intvector.go new file mode 100644 index 000000000..5ad9e294b --- /dev/null +++ b/libgo/go/container/vector/intvector.go @@ -0,0 +1,208 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// CAUTION: If this file is not vector.go, it was generated +// automatically from vector.go - DO NOT EDIT in that case! + +package vector + + +func (p *IntVector) realloc(length, capacity int) (b []int) { + if capacity < initialSize { + capacity = initialSize + } + if capacity < length { + capacity = length + } + b = make(IntVector, length, capacity) + copy(b, *p) + *p = b + return +} + + +// Insert n elements at position i. +func (p *IntVector) Expand(i, n int) { + a := *p + + // make sure we have enough space + len0 := len(a) + len1 := len0 + n + if len1 <= cap(a) { + // enough space - just expand + a = a[0:len1] + } else { + // not enough space - double capacity + capb := cap(a) * 2 + if capb < len1 { + // still not enough - use required length + capb = len1 + } + // capb >= len1 + a = p.realloc(len1, capb) + } + + // make a hole + for j := len0 - 1; j >= i; j-- { + a[j+n] = a[j] + } + + *p = a +} + + +// Insert n elements at the end of a vector. +func (p *IntVector) Extend(n int) { p.Expand(len(*p), n) } + + +// Resize changes the length and capacity of a vector. +// If the new length is shorter than the current length, Resize discards +// trailing elements. If the new length is longer than the current length, +// Resize adds the respective zero values for the additional elements. The capacity +// parameter is ignored unless the new length or capacity is longer than the current +// capacity. The resized vector's capacity may be larger than the requested capacity. +func (p *IntVector) Resize(length, capacity int) *IntVector { + a := *p + + if length > cap(a) || capacity > cap(a) { + // not enough space or larger capacity requested explicitly + a = p.realloc(length, capacity) + } else if length < len(a) { + // clear trailing elements + for i := range a[length:] { + var zero int + a[length+i] = zero + } + } + + *p = a[0:length] + return p +} + + +// Len returns the number of elements in the vector. +// Same as len(*p). +func (p *IntVector) Len() int { return len(*p) } + + +// Cap returns the capacity of the vector; that is, the +// maximum length the vector can grow without resizing. +// Same as cap(*p). +func (p *IntVector) Cap() int { return cap(*p) } + + +// At returns the i'th element of the vector. +func (p *IntVector) At(i int) int { return (*p)[i] } + + +// Set sets the i'th element of the vector to value x. +func (p *IntVector) Set(i int, x int) { (*p)[i] = x } + + +// Last returns the element in the vector of highest index. +func (p *IntVector) Last() int { return (*p)[len(*p)-1] } + + +// Copy makes a copy of the vector and returns it. +func (p *IntVector) Copy() IntVector { + arr := make(IntVector, len(*p)) + copy(arr, *p) + return arr +} + + +// Insert inserts into the vector an element of value x before +// the current element at index i. +func (p *IntVector) Insert(i int, x int) { + p.Expand(i, 1) + (*p)[i] = x +} + + +// Delete deletes the i'th element of the vector. The gap is closed so the old +// element at index i+1 has index i afterwards. +func (p *IntVector) Delete(i int) { + a := *p + n := len(a) + + copy(a[i:n-1], a[i+1:n]) + var zero int + a[n-1] = zero // support GC, zero out entry + *p = a[0 : n-1] +} + + +// InsertVector inserts into the vector the contents of the vector +// x such that the 0th element of x appears at index i after insertion. +func (p *IntVector) InsertVector(i int, x *IntVector) { + b := *x + + p.Expand(i, len(b)) + copy((*p)[i:i+len(b)], b) +} + + +// Cut deletes elements i through j-1, inclusive. +func (p *IntVector) Cut(i, j int) { + a := *p + n := len(a) + m := n - (j - i) + + copy(a[i:m], a[j:n]) + for k := m; k < n; k++ { //TODO(bflm) don't zero out the elements unless it's a Vector. + var zero int + a[k] = zero // support GC, zero out entries + } + + *p = a[0:m] +} + + +// Slice returns a new sub-vector by slicing the old one to extract slice [i:j]. +// The elements are copied. The original vector is unchanged. +func (p *IntVector) Slice(i, j int) *IntVector { + var s IntVector + s.realloc(j-i, 0) // will fail in Init() if j < i + copy(s, (*p)[i:j]) + return &s +} + + +// Convenience wrappers + +// Push appends x to the end of the vector. +func (p *IntVector) Push(x int) { p.Insert(len(*p), x) } + + +// Pop deletes the last element of the vector. +func (p *IntVector) Pop() int { + a := *p + + i := len(a) - 1 + x := a[i] + var zero int + a[i] = zero // support GC, zero out entry + *p = a[0:i] + return x +} + + +// AppendVector appends the entire vector x to the end of this vector. +func (p *IntVector) AppendVector(x *IntVector) { p.InsertVector(len(*p), x) } + + +// Swap exchanges the elements at indexes i and j. +func (p *IntVector) Swap(i, j int) { + a := *p + a[i], a[j] = a[j], a[i] +} + + +// Do calls function f for each element of the vector, in order. +// The behavior of Do is undefined if f changes *p. +func (p *IntVector) Do(f func(elem int)) { + for _, e := range *p { + f(e) + } +} diff --git a/libgo/go/container/vector/intvector_test.go b/libgo/go/container/vector/intvector_test.go new file mode 100644 index 000000000..1e38a1982 --- /dev/null +++ b/libgo/go/container/vector/intvector_test.go @@ -0,0 +1,344 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// CAUTION: If this file is not vector_test.go, it was generated +// automatically from vector_test.go - DO NOT EDIT in that case! + +package vector + +import "testing" + + +func TestIntZeroLen(t *testing.T) { + a := new(IntVector) + if a.Len() != 0 { + t.Errorf("%T: B1) expected 0, got %d", a, a.Len()) + } + if len(*a) != 0 { + t.Errorf("%T: B2) expected 0, got %d", a, len(*a)) + } + var b IntVector + if b.Len() != 0 { + t.Errorf("%T: B3) expected 0, got %d", b, b.Len()) + } + if len(b) != 0 { + t.Errorf("%T: B4) expected 0, got %d", b, len(b)) + } +} + + +func TestIntResize(t *testing.T) { + var a IntVector + checkSize(t, &a, 0, 0) + checkSize(t, a.Resize(0, 5), 0, 5) + checkSize(t, a.Resize(1, 0), 1, 5) + checkSize(t, a.Resize(10, 0), 10, 10) + checkSize(t, a.Resize(5, 0), 5, 10) + checkSize(t, a.Resize(3, 8), 3, 10) + checkSize(t, a.Resize(0, 100), 0, 100) + checkSize(t, a.Resize(11, 100), 11, 100) +} + + +func TestIntResize2(t *testing.T) { + var a IntVector + checkSize(t, &a, 0, 0) + a.Push(int2IntValue(1)) + a.Push(int2IntValue(2)) + a.Push(int2IntValue(3)) + a.Push(int2IntValue(4)) + checkSize(t, &a, 4, 4) + checkSize(t, a.Resize(10, 0), 10, 10) + for i := 4; i < a.Len(); i++ { + if a.At(i) != intzero { + t.Errorf("%T: expected a.At(%d) == %v; found %v!", a, i, intzero, a.At(i)) + } + } + for i := 4; i < len(a); i++ { + if a[i] != intzero { + t.Errorf("%T: expected a[%d] == %v; found %v", a, i, intzero, a[i]) + } + } +} + + +func checkIntZero(t *testing.T, a *IntVector, i int) { + for j := 0; j < i; j++ { + if a.At(j) == intzero { + t.Errorf("%T: 1 expected a.At(%d) == %d; found %v", a, j, j, a.At(j)) + } + if (*a)[j] == intzero { + t.Errorf("%T: 2 expected (*a)[%d] == %d; found %v", a, j, j, (*a)[j]) + } + } + for ; i < a.Len(); i++ { + if a.At(i) != intzero { + t.Errorf("%T: 3 expected a.At(%d) == %v; found %v", a, i, intzero, a.At(i)) + } + if (*a)[i] != intzero { + t.Errorf("%T: 4 expected (*a)[%d] == %v; found %v", a, i, intzero, (*a)[i]) + } + } +} + + +func TestIntTrailingElements(t *testing.T) { + var a IntVector + for i := 0; i < 10; i++ { + a.Push(int2IntValue(i + 1)) + } + checkIntZero(t, &a, 10) + checkSize(t, &a, 10, 16) + checkSize(t, a.Resize(5, 0), 5, 16) + checkSize(t, a.Resize(10, 0), 10, 16) + checkIntZero(t, &a, 5) +} + + +func TestIntAccess(t *testing.T) { + const n = 100 + var a IntVector + a.Resize(n, 0) + for i := 0; i < n; i++ { + a.Set(i, int2IntValue(val(i))) + } + for i := 0; i < n; i++ { + if elem2IntValue(a.At(i)) != int2IntValue(val(i)) { + t.Error(i) + } + } + var b IntVector + b.Resize(n, 0) + for i := 0; i < n; i++ { + b[i] = int2IntValue(val(i)) + } + for i := 0; i < n; i++ { + if elem2IntValue(b[i]) != int2IntValue(val(i)) { + t.Error(i) + } + } +} + + +func TestIntInsertDeleteClear(t *testing.T) { + const n = 100 + var a IntVector + + for i := 0; i < n; i++ { + if a.Len() != i { + t.Errorf("%T: A) wrong Len() %d (expected %d)", a, a.Len(), i) + } + if len(a) != i { + t.Errorf("%T: A) wrong len() %d (expected %d)", a, len(a), i) + } + a.Insert(0, int2IntValue(val(i))) + if elem2IntValue(a.Last()) != int2IntValue(val(0)) { + t.Errorf("%T: B", a) + } + } + for i := n - 1; i >= 0; i-- { + if elem2IntValue(a.Last()) != int2IntValue(val(0)) { + t.Errorf("%T: C", a) + } + if elem2IntValue(a.At(0)) != int2IntValue(val(i)) { + t.Errorf("%T: D", a) + } + if elem2IntValue(a[0]) != int2IntValue(val(i)) { + t.Errorf("%T: D2", a) + } + a.Delete(0) + if a.Len() != i { + t.Errorf("%T: E) wrong Len() %d (expected %d)", a, a.Len(), i) + } + if len(a) != i { + t.Errorf("%T: E) wrong len() %d (expected %d)", a, len(a), i) + } + } + + if a.Len() != 0 { + t.Errorf("%T: F) wrong Len() %d (expected 0)", a, a.Len()) + } + if len(a) != 0 { + t.Errorf("%T: F) wrong len() %d (expected 0)", a, len(a)) + } + for i := 0; i < n; i++ { + a.Push(int2IntValue(val(i))) + if a.Len() != i+1 { + t.Errorf("%T: G) wrong Len() %d (expected %d)", a, a.Len(), i+1) + } + if len(a) != i+1 { + t.Errorf("%T: G) wrong len() %d (expected %d)", a, len(a), i+1) + } + if elem2IntValue(a.Last()) != int2IntValue(val(i)) { + t.Errorf("%T: H", a) + } + } + a.Resize(0, 0) + if a.Len() != 0 { + t.Errorf("%T: I wrong Len() %d (expected 0)", a, a.Len()) + } + if len(a) != 0 { + t.Errorf("%T: I wrong len() %d (expected 0)", a, len(a)) + } + + const m = 5 + for j := 0; j < m; j++ { + a.Push(int2IntValue(j)) + for i := 0; i < n; i++ { + x := val(i) + a.Push(int2IntValue(x)) + if elem2IntValue(a.Pop()) != int2IntValue(x) { + t.Errorf("%T: J", a) + } + if a.Len() != j+1 { + t.Errorf("%T: K) wrong Len() %d (expected %d)", a, a.Len(), j+1) + } + if len(a) != j+1 { + t.Errorf("%T: K) wrong len() %d (expected %d)", a, len(a), j+1) + } + } + } + if a.Len() != m { + t.Errorf("%T: L) wrong Len() %d (expected %d)", a, a.Len(), m) + } + if len(a) != m { + t.Errorf("%T: L) wrong len() %d (expected %d)", a, len(a), m) + } +} + + +func verify_sliceInt(t *testing.T, x *IntVector, elt, i, j int) { + for k := i; k < j; k++ { + if elem2IntValue(x.At(k)) != int2IntValue(elt) { + t.Errorf("%T: M) wrong [%d] element %v (expected %v)", x, k, elem2IntValue(x.At(k)), int2IntValue(elt)) + } + } + + s := x.Slice(i, j) + for k, n := 0, j-i; k < n; k++ { + if elem2IntValue(s.At(k)) != int2IntValue(elt) { + t.Errorf("%T: N) wrong [%d] element %v (expected %v)", x, k, elem2IntValue(x.At(k)), int2IntValue(elt)) + } + } +} + + +func verify_patternInt(t *testing.T, x *IntVector, a, b, c int) { + n := a + b + c + if x.Len() != n { + t.Errorf("%T: O) wrong Len() %d (expected %d)", x, x.Len(), n) + } + if len(*x) != n { + t.Errorf("%T: O) wrong len() %d (expected %d)", x, len(*x), n) + } + verify_sliceInt(t, x, 0, 0, a) + verify_sliceInt(t, x, 1, a, a+b) + verify_sliceInt(t, x, 0, a+b, n) +} + + +func make_vectorInt(elt, len int) *IntVector { + x := new(IntVector).Resize(len, 0) + for i := 0; i < len; i++ { + x.Set(i, int2IntValue(elt)) + } + return x +} + + +func TestIntInsertVector(t *testing.T) { + // 1 + a := make_vectorInt(0, 0) + b := make_vectorInt(1, 10) + a.InsertVector(0, b) + verify_patternInt(t, a, 0, 10, 0) + // 2 + a = make_vectorInt(0, 10) + b = make_vectorInt(1, 0) + a.InsertVector(5, b) + verify_patternInt(t, a, 5, 0, 5) + // 3 + a = make_vectorInt(0, 10) + b = make_vectorInt(1, 3) + a.InsertVector(3, b) + verify_patternInt(t, a, 3, 3, 7) + // 4 + a = make_vectorInt(0, 10) + b = make_vectorInt(1, 1000) + a.InsertVector(8, b) + verify_patternInt(t, a, 8, 1000, 2) +} + + +func TestIntDo(t *testing.T) { + const n = 25 + const salt = 17 + a := new(IntVector).Resize(n, 0) + for i := 0; i < n; i++ { + a.Set(i, int2IntValue(salt*i)) + } + count := 0 + a.Do(func(e int) { + i := intf2IntValue(e) + if i != int2IntValue(count*salt) { + t.Error(tname(a), "value at", count, "should be", count*salt, "not", i) + } + count++ + }) + if count != n { + t.Error(tname(a), "should visit", n, "values; did visit", count) + } + + b := new(IntVector).Resize(n, 0) + for i := 0; i < n; i++ { + (*b)[i] = int2IntValue(salt * i) + } + count = 0 + b.Do(func(e int) { + i := intf2IntValue(e) + if i != int2IntValue(count*salt) { + t.Error(tname(b), "b) value at", count, "should be", count*salt, "not", i) + } + count++ + }) + if count != n { + t.Error(tname(b), "b) should visit", n, "values; did visit", count) + } + + var c IntVector + c.Resize(n, 0) + for i := 0; i < n; i++ { + c[i] = int2IntValue(salt * i) + } + count = 0 + c.Do(func(e int) { + i := intf2IntValue(e) + if i != int2IntValue(count*salt) { + t.Error(tname(c), "c) value at", count, "should be", count*salt, "not", i) + } + count++ + }) + if count != n { + t.Error(tname(c), "c) should visit", n, "values; did visit", count) + } + +} + + +func TestIntVectorCopy(t *testing.T) { + // verify Copy() returns a copy, not simply a slice of the original vector + const Len = 10 + var src IntVector + for i := 0; i < Len; i++ { + src.Push(int2IntValue(i * i)) + } + dest := src.Copy() + for i := 0; i < Len; i++ { + src[i] = int2IntValue(-1) + v := elem2IntValue(dest[i]) + if v != int2IntValue(i*i) { + t.Error(tname(src), "expected", i*i, "got", v) + } + } +} diff --git a/libgo/go/container/vector/nogen_test.go b/libgo/go/container/vector/nogen_test.go new file mode 100644 index 000000000..790d3749f --- /dev/null +++ b/libgo/go/container/vector/nogen_test.go @@ -0,0 +1,76 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package vector + + +import ( + "fmt" + "sort" + "testing" +) + +var ( + zero interface{} + intzero int + strzero string +) + + +func int2Value(x int) int { return x } +func int2IntValue(x int) int { return x } +func int2StrValue(x int) string { return string(x) } + + +func elem2Value(x interface{}) int { return x.(int) } +func elem2IntValue(x int) int { return x } +func elem2StrValue(x string) string { return x } + + +func intf2Value(x interface{}) int { return x.(int) } +func intf2IntValue(x interface{}) int { return x.(int) } +func intf2StrValue(x interface{}) string { return x.(string) } + + +type VectorInterface interface { + Len() int + Cap() int +} + + +func checkSize(t *testing.T, v VectorInterface, len, cap int) { + if v.Len() != len { + t.Errorf("%T expected len = %d; found %d", v, len, v.Len()) + } + if v.Cap() < cap { + t.Errorf("%T expected cap >= %d; found %d", v, cap, v.Cap()) + } +} + + +func val(i int) int { return i*991 - 1234 } + + +func TestSorting(t *testing.T) { + const n = 100 + + a := new(IntVector).Resize(n, 0) + for i := n - 1; i >= 0; i-- { + a.Set(i, n-1-i) + } + if sort.IsSorted(a) { + t.Error("int vector not sorted") + } + + b := new(StringVector).Resize(n, 0) + for i := n - 1; i >= 0; i-- { + b.Set(i, fmt.Sprint(n-1-i)) + } + if sort.IsSorted(b) { + t.Error("string vector not sorted") + } +} + + +func tname(x interface{}) string { return fmt.Sprintf("%T: ", x) } diff --git a/libgo/go/container/vector/numbers_test.go b/libgo/go/container/vector/numbers_test.go new file mode 100644 index 000000000..d540ace05 --- /dev/null +++ b/libgo/go/container/vector/numbers_test.go @@ -0,0 +1,122 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package vector + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + + +const memTestN = 1000000 + + +func s(n uint64) string { + str := fmt.Sprintf("%d", n) + lens := len(str) + a := make([]string, (lens+2)/3) + start := lens + for i := range a { + start -= 3 + if start < 0 { + start = 0 + } + a[len(a)-i-1] = str[start:lens] + lens -= 3 + } + return strings.Join(a, " ") +} + + +func TestVectorNums(t *testing.T) { + var v Vector + c := int(0) + runtime.GC() + m0 := runtime.MemStats + v.Resize(memTestN, memTestN) + for i := 0; i < memTestN; i++ { + v.Set(i, c) + } + runtime.GC() + m := runtime.MemStats + v.Resize(0, 0) + runtime.GC() + n := m.Alloc - m0.Alloc + t.Logf("%T.Push(%#v), n = %s: Alloc/n = %.2f\n", v, c, s(memTestN), float64(n)/memTestN) +} + + +func TestIntVectorNums(t *testing.T) { + var v IntVector + c := int(0) + runtime.GC() + m0 := runtime.MemStats + v.Resize(memTestN, memTestN) + for i := 0; i < memTestN; i++ { + v.Set(i, c) + } + runtime.GC() + m := runtime.MemStats + v.Resize(0, 0) + runtime.GC() + n := m.Alloc - m0.Alloc + t.Logf("%T.Push(%#v), n = %s: Alloc/n = %.2f\n", v, c, s(memTestN), float64(n)/memTestN) +} + + +func TestStringVectorNums(t *testing.T) { + var v StringVector + c := "" + runtime.GC() + m0 := runtime.MemStats + v.Resize(memTestN, memTestN) + for i := 0; i < memTestN; i++ { + v.Set(i, c) + } + runtime.GC() + m := runtime.MemStats + v.Resize(0, 0) + runtime.GC() + n := m.Alloc - m0.Alloc + t.Logf("%T.Push(%#v), n = %s: Alloc/n = %.2f\n", v, c, s(memTestN), float64(n)/memTestN) +} + + +func BenchmarkVectorNums(b *testing.B) { + c := int(0) + var v Vector + b.StopTimer() + runtime.GC() + b.StartTimer() + for i := 0; i < b.N; i++ { + v.Push(c) + } +} + + +func BenchmarkIntVectorNums(b *testing.B) { + c := int(0) + var v IntVector + b.StopTimer() + runtime.GC() + b.StartTimer() + for i := 0; i < b.N; i++ { + v.Push(c) + } +} + + +func BenchmarkStringVectorNums(b *testing.B) { + c := "" + var v StringVector + b.StopTimer() + runtime.GC() + b.StartTimer() + for i := 0; i < b.N; i++ { + v.Push(c) + } +} diff --git a/libgo/go/container/vector/stringvector.go b/libgo/go/container/vector/stringvector.go new file mode 100644 index 000000000..852685f5a --- /dev/null +++ b/libgo/go/container/vector/stringvector.go @@ -0,0 +1,208 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// CAUTION: If this file is not vector.go, it was generated +// automatically from vector.go - DO NOT EDIT in that case! + +package vector + + +func (p *StringVector) realloc(length, capacity int) (b []string) { + if capacity < initialSize { + capacity = initialSize + } + if capacity < length { + capacity = length + } + b = make(StringVector, length, capacity) + copy(b, *p) + *p = b + return +} + + +// Insert n elements at position i. +func (p *StringVector) Expand(i, n int) { + a := *p + + // make sure we have enough space + len0 := len(a) + len1 := len0 + n + if len1 <= cap(a) { + // enough space - just expand + a = a[0:len1] + } else { + // not enough space - double capacity + capb := cap(a) * 2 + if capb < len1 { + // still not enough - use required length + capb = len1 + } + // capb >= len1 + a = p.realloc(len1, capb) + } + + // make a hole + for j := len0 - 1; j >= i; j-- { + a[j+n] = a[j] + } + + *p = a +} + + +// Insert n elements at the end of a vector. +func (p *StringVector) Extend(n int) { p.Expand(len(*p), n) } + + +// Resize changes the length and capacity of a vector. +// If the new length is shorter than the current length, Resize discards +// trailing elements. If the new length is longer than the current length, +// Resize adds the respective zero values for the additional elements. The capacity +// parameter is ignored unless the new length or capacity is longer than the current +// capacity. The resized vector's capacity may be larger than the requested capacity. +func (p *StringVector) Resize(length, capacity int) *StringVector { + a := *p + + if length > cap(a) || capacity > cap(a) { + // not enough space or larger capacity requested explicitly + a = p.realloc(length, capacity) + } else if length < len(a) { + // clear trailing elements + for i := range a[length:] { + var zero string + a[length+i] = zero + } + } + + *p = a[0:length] + return p +} + + +// Len returns the number of elements in the vector. +// Same as len(*p). +func (p *StringVector) Len() int { return len(*p) } + + +// Cap returns the capacity of the vector; that is, the +// maximum length the vector can grow without resizing. +// Same as cap(*p). +func (p *StringVector) Cap() int { return cap(*p) } + + +// At returns the i'th element of the vector. +func (p *StringVector) At(i int) string { return (*p)[i] } + + +// Set sets the i'th element of the vector to value x. +func (p *StringVector) Set(i int, x string) { (*p)[i] = x } + + +// Last returns the element in the vector of highest index. +func (p *StringVector) Last() string { return (*p)[len(*p)-1] } + + +// Copy makes a copy of the vector and returns it. +func (p *StringVector) Copy() StringVector { + arr := make(StringVector, len(*p)) + copy(arr, *p) + return arr +} + + +// Insert inserts into the vector an element of value x before +// the current element at index i. +func (p *StringVector) Insert(i int, x string) { + p.Expand(i, 1) + (*p)[i] = x +} + + +// Delete deletes the i'th element of the vector. The gap is closed so the old +// element at index i+1 has index i afterwards. +func (p *StringVector) Delete(i int) { + a := *p + n := len(a) + + copy(a[i:n-1], a[i+1:n]) + var zero string + a[n-1] = zero // support GC, zero out entry + *p = a[0 : n-1] +} + + +// InsertVector inserts into the vector the contents of the vector +// x such that the 0th element of x appears at index i after insertion. +func (p *StringVector) InsertVector(i int, x *StringVector) { + b := *x + + p.Expand(i, len(b)) + copy((*p)[i:i+len(b)], b) +} + + +// Cut deletes elements i through j-1, inclusive. +func (p *StringVector) Cut(i, j int) { + a := *p + n := len(a) + m := n - (j - i) + + copy(a[i:m], a[j:n]) + for k := m; k < n; k++ { //TODO(bflm) don't zero out the elements unless it's a Vector. + var zero string + a[k] = zero // support GC, zero out entries + } + + *p = a[0:m] +} + + +// Slice returns a new sub-vector by slicing the old one to extract slice [i:j]. +// The elements are copied. The original vector is unchanged. +func (p *StringVector) Slice(i, j int) *StringVector { + var s StringVector + s.realloc(j-i, 0) // will fail in Init() if j < i + copy(s, (*p)[i:j]) + return &s +} + + +// Convenience wrappers + +// Push appends x to the end of the vector. +func (p *StringVector) Push(x string) { p.Insert(len(*p), x) } + + +// Pop deletes the last element of the vector. +func (p *StringVector) Pop() string { + a := *p + + i := len(a) - 1 + x := a[i] + var zero string + a[i] = zero // support GC, zero out entry + *p = a[0:i] + return x +} + + +// AppendVector appends the entire vector x to the end of this vector. +func (p *StringVector) AppendVector(x *StringVector) { p.InsertVector(len(*p), x) } + + +// Swap exchanges the elements at indexes i and j. +func (p *StringVector) Swap(i, j int) { + a := *p + a[i], a[j] = a[j], a[i] +} + + +// Do calls function f for each element of the vector, in order. +// The behavior of Do is undefined if f changes *p. +func (p *StringVector) Do(f func(elem string)) { + for _, e := range *p { + f(e) + } +} diff --git a/libgo/go/container/vector/stringvector_test.go b/libgo/go/container/vector/stringvector_test.go new file mode 100644 index 000000000..776ae26de --- /dev/null +++ b/libgo/go/container/vector/stringvector_test.go @@ -0,0 +1,344 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// CAUTION: If this file is not vector_test.go, it was generated +// automatically from vector_test.go - DO NOT EDIT in that case! + +package vector + +import "testing" + + +func TestStrZeroLen(t *testing.T) { + a := new(StringVector) + if a.Len() != 0 { + t.Errorf("%T: B1) expected 0, got %d", a, a.Len()) + } + if len(*a) != 0 { + t.Errorf("%T: B2) expected 0, got %d", a, len(*a)) + } + var b StringVector + if b.Len() != 0 { + t.Errorf("%T: B3) expected 0, got %d", b, b.Len()) + } + if len(b) != 0 { + t.Errorf("%T: B4) expected 0, got %d", b, len(b)) + } +} + + +func TestStrResize(t *testing.T) { + var a StringVector + checkSize(t, &a, 0, 0) + checkSize(t, a.Resize(0, 5), 0, 5) + checkSize(t, a.Resize(1, 0), 1, 5) + checkSize(t, a.Resize(10, 0), 10, 10) + checkSize(t, a.Resize(5, 0), 5, 10) + checkSize(t, a.Resize(3, 8), 3, 10) + checkSize(t, a.Resize(0, 100), 0, 100) + checkSize(t, a.Resize(11, 100), 11, 100) +} + + +func TestStrResize2(t *testing.T) { + var a StringVector + checkSize(t, &a, 0, 0) + a.Push(int2StrValue(1)) + a.Push(int2StrValue(2)) + a.Push(int2StrValue(3)) + a.Push(int2StrValue(4)) + checkSize(t, &a, 4, 4) + checkSize(t, a.Resize(10, 0), 10, 10) + for i := 4; i < a.Len(); i++ { + if a.At(i) != strzero { + t.Errorf("%T: expected a.At(%d) == %v; found %v!", a, i, strzero, a.At(i)) + } + } + for i := 4; i < len(a); i++ { + if a[i] != strzero { + t.Errorf("%T: expected a[%d] == %v; found %v", a, i, strzero, a[i]) + } + } +} + + +func checkStrZero(t *testing.T, a *StringVector, i int) { + for j := 0; j < i; j++ { + if a.At(j) == strzero { + t.Errorf("%T: 1 expected a.At(%d) == %d; found %v", a, j, j, a.At(j)) + } + if (*a)[j] == strzero { + t.Errorf("%T: 2 expected (*a)[%d] == %d; found %v", a, j, j, (*a)[j]) + } + } + for ; i < a.Len(); i++ { + if a.At(i) != strzero { + t.Errorf("%T: 3 expected a.At(%d) == %v; found %v", a, i, strzero, a.At(i)) + } + if (*a)[i] != strzero { + t.Errorf("%T: 4 expected (*a)[%d] == %v; found %v", a, i, strzero, (*a)[i]) + } + } +} + + +func TestStrTrailingElements(t *testing.T) { + var a StringVector + for i := 0; i < 10; i++ { + a.Push(int2StrValue(i + 1)) + } + checkStrZero(t, &a, 10) + checkSize(t, &a, 10, 16) + checkSize(t, a.Resize(5, 0), 5, 16) + checkSize(t, a.Resize(10, 0), 10, 16) + checkStrZero(t, &a, 5) +} + + +func TestStrAccess(t *testing.T) { + const n = 100 + var a StringVector + a.Resize(n, 0) + for i := 0; i < n; i++ { + a.Set(i, int2StrValue(val(i))) + } + for i := 0; i < n; i++ { + if elem2StrValue(a.At(i)) != int2StrValue(val(i)) { + t.Error(i) + } + } + var b StringVector + b.Resize(n, 0) + for i := 0; i < n; i++ { + b[i] = int2StrValue(val(i)) + } + for i := 0; i < n; i++ { + if elem2StrValue(b[i]) != int2StrValue(val(i)) { + t.Error(i) + } + } +} + + +func TestStrInsertDeleteClear(t *testing.T) { + const n = 100 + var a StringVector + + for i := 0; i < n; i++ { + if a.Len() != i { + t.Errorf("%T: A) wrong Len() %d (expected %d)", a, a.Len(), i) + } + if len(a) != i { + t.Errorf("%T: A) wrong len() %d (expected %d)", a, len(a), i) + } + a.Insert(0, int2StrValue(val(i))) + if elem2StrValue(a.Last()) != int2StrValue(val(0)) { + t.Errorf("%T: B", a) + } + } + for i := n - 1; i >= 0; i-- { + if elem2StrValue(a.Last()) != int2StrValue(val(0)) { + t.Errorf("%T: C", a) + } + if elem2StrValue(a.At(0)) != int2StrValue(val(i)) { + t.Errorf("%T: D", a) + } + if elem2StrValue(a[0]) != int2StrValue(val(i)) { + t.Errorf("%T: D2", a) + } + a.Delete(0) + if a.Len() != i { + t.Errorf("%T: E) wrong Len() %d (expected %d)", a, a.Len(), i) + } + if len(a) != i { + t.Errorf("%T: E) wrong len() %d (expected %d)", a, len(a), i) + } + } + + if a.Len() != 0 { + t.Errorf("%T: F) wrong Len() %d (expected 0)", a, a.Len()) + } + if len(a) != 0 { + t.Errorf("%T: F) wrong len() %d (expected 0)", a, len(a)) + } + for i := 0; i < n; i++ { + a.Push(int2StrValue(val(i))) + if a.Len() != i+1 { + t.Errorf("%T: G) wrong Len() %d (expected %d)", a, a.Len(), i+1) + } + if len(a) != i+1 { + t.Errorf("%T: G) wrong len() %d (expected %d)", a, len(a), i+1) + } + if elem2StrValue(a.Last()) != int2StrValue(val(i)) { + t.Errorf("%T: H", a) + } + } + a.Resize(0, 0) + if a.Len() != 0 { + t.Errorf("%T: I wrong Len() %d (expected 0)", a, a.Len()) + } + if len(a) != 0 { + t.Errorf("%T: I wrong len() %d (expected 0)", a, len(a)) + } + + const m = 5 + for j := 0; j < m; j++ { + a.Push(int2StrValue(j)) + for i := 0; i < n; i++ { + x := val(i) + a.Push(int2StrValue(x)) + if elem2StrValue(a.Pop()) != int2StrValue(x) { + t.Errorf("%T: J", a) + } + if a.Len() != j+1 { + t.Errorf("%T: K) wrong Len() %d (expected %d)", a, a.Len(), j+1) + } + if len(a) != j+1 { + t.Errorf("%T: K) wrong len() %d (expected %d)", a, len(a), j+1) + } + } + } + if a.Len() != m { + t.Errorf("%T: L) wrong Len() %d (expected %d)", a, a.Len(), m) + } + if len(a) != m { + t.Errorf("%T: L) wrong len() %d (expected %d)", a, len(a), m) + } +} + + +func verify_sliceStr(t *testing.T, x *StringVector, elt, i, j int) { + for k := i; k < j; k++ { + if elem2StrValue(x.At(k)) != int2StrValue(elt) { + t.Errorf("%T: M) wrong [%d] element %v (expected %v)", x, k, elem2StrValue(x.At(k)), int2StrValue(elt)) + } + } + + s := x.Slice(i, j) + for k, n := 0, j-i; k < n; k++ { + if elem2StrValue(s.At(k)) != int2StrValue(elt) { + t.Errorf("%T: N) wrong [%d] element %v (expected %v)", x, k, elem2StrValue(x.At(k)), int2StrValue(elt)) + } + } +} + + +func verify_patternStr(t *testing.T, x *StringVector, a, b, c int) { + n := a + b + c + if x.Len() != n { + t.Errorf("%T: O) wrong Len() %d (expected %d)", x, x.Len(), n) + } + if len(*x) != n { + t.Errorf("%T: O) wrong len() %d (expected %d)", x, len(*x), n) + } + verify_sliceStr(t, x, 0, 0, a) + verify_sliceStr(t, x, 1, a, a+b) + verify_sliceStr(t, x, 0, a+b, n) +} + + +func make_vectorStr(elt, len int) *StringVector { + x := new(StringVector).Resize(len, 0) + for i := 0; i < len; i++ { + x.Set(i, int2StrValue(elt)) + } + return x +} + + +func TestStrInsertVector(t *testing.T) { + // 1 + a := make_vectorStr(0, 0) + b := make_vectorStr(1, 10) + a.InsertVector(0, b) + verify_patternStr(t, a, 0, 10, 0) + // 2 + a = make_vectorStr(0, 10) + b = make_vectorStr(1, 0) + a.InsertVector(5, b) + verify_patternStr(t, a, 5, 0, 5) + // 3 + a = make_vectorStr(0, 10) + b = make_vectorStr(1, 3) + a.InsertVector(3, b) + verify_patternStr(t, a, 3, 3, 7) + // 4 + a = make_vectorStr(0, 10) + b = make_vectorStr(1, 1000) + a.InsertVector(8, b) + verify_patternStr(t, a, 8, 1000, 2) +} + + +func TestStrDo(t *testing.T) { + const n = 25 + const salt = 17 + a := new(StringVector).Resize(n, 0) + for i := 0; i < n; i++ { + a.Set(i, int2StrValue(salt*i)) + } + count := 0 + a.Do(func(e string) { + i := intf2StrValue(e) + if i != int2StrValue(count*salt) { + t.Error(tname(a), "value at", count, "should be", count*salt, "not", i) + } + count++ + }) + if count != n { + t.Error(tname(a), "should visit", n, "values; did visit", count) + } + + b := new(StringVector).Resize(n, 0) + for i := 0; i < n; i++ { + (*b)[i] = int2StrValue(salt * i) + } + count = 0 + b.Do(func(e string) { + i := intf2StrValue(e) + if i != int2StrValue(count*salt) { + t.Error(tname(b), "b) value at", count, "should be", count*salt, "not", i) + } + count++ + }) + if count != n { + t.Error(tname(b), "b) should visit", n, "values; did visit", count) + } + + var c StringVector + c.Resize(n, 0) + for i := 0; i < n; i++ { + c[i] = int2StrValue(salt * i) + } + count = 0 + c.Do(func(e string) { + i := intf2StrValue(e) + if i != int2StrValue(count*salt) { + t.Error(tname(c), "c) value at", count, "should be", count*salt, "not", i) + } + count++ + }) + if count != n { + t.Error(tname(c), "c) should visit", n, "values; did visit", count) + } + +} + + +func TestStrVectorCopy(t *testing.T) { + // verify Copy() returns a copy, not simply a slice of the original vector + const Len = 10 + var src StringVector + for i := 0; i < Len; i++ { + src.Push(int2StrValue(i * i)) + } + dest := src.Copy() + for i := 0; i < Len; i++ { + src[i] = int2StrValue(-1) + v := elem2StrValue(dest[i]) + if v != int2StrValue(i*i) { + t.Error(tname(src), "expected", i*i, "got", v) + } + } +} diff --git a/libgo/go/container/vector/vector.go b/libgo/go/container/vector/vector.go new file mode 100644 index 000000000..f43e4d23c --- /dev/null +++ b/libgo/go/container/vector/vector.go @@ -0,0 +1,208 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// CAUTION: If this file is not vector.go, it was generated +// automatically from vector.go - DO NOT EDIT in that case! + +package vector + + +func (p *Vector) realloc(length, capacity int) (b []interface{}) { + if capacity < initialSize { + capacity = initialSize + } + if capacity < length { + capacity = length + } + b = make(Vector, length, capacity) + copy(b, *p) + *p = b + return +} + + +// Insert n elements at position i. +func (p *Vector) Expand(i, n int) { + a := *p + + // make sure we have enough space + len0 := len(a) + len1 := len0 + n + if len1 <= cap(a) { + // enough space - just expand + a = a[0:len1] + } else { + // not enough space - double capacity + capb := cap(a) * 2 + if capb < len1 { + // still not enough - use required length + capb = len1 + } + // capb >= len1 + a = p.realloc(len1, capb) + } + + // make a hole + for j := len0 - 1; j >= i; j-- { + a[j+n] = a[j] + } + + *p = a +} + + +// Insert n elements at the end of a vector. +func (p *Vector) Extend(n int) { p.Expand(len(*p), n) } + + +// Resize changes the length and capacity of a vector. +// If the new length is shorter than the current length, Resize discards +// trailing elements. If the new length is longer than the current length, +// Resize adds the respective zero values for the additional elements. The capacity +// parameter is ignored unless the new length or capacity is longer than the current +// capacity. The resized vector's capacity may be larger than the requested capacity. +func (p *Vector) Resize(length, capacity int) *Vector { + a := *p + + if length > cap(a) || capacity > cap(a) { + // not enough space or larger capacity requested explicitly + a = p.realloc(length, capacity) + } else if length < len(a) { + // clear trailing elements + for i := range a[length:] { + var zero interface{} + a[length+i] = zero + } + } + + *p = a[0:length] + return p +} + + +// Len returns the number of elements in the vector. +// Same as len(*p). +func (p *Vector) Len() int { return len(*p) } + + +// Cap returns the capacity of the vector; that is, the +// maximum length the vector can grow without resizing. +// Same as cap(*p). +func (p *Vector) Cap() int { return cap(*p) } + + +// At returns the i'th element of the vector. +func (p *Vector) At(i int) interface{} { return (*p)[i] } + + +// Set sets the i'th element of the vector to value x. +func (p *Vector) Set(i int, x interface{}) { (*p)[i] = x } + + +// Last returns the element in the vector of highest index. +func (p *Vector) Last() interface{} { return (*p)[len(*p)-1] } + + +// Copy makes a copy of the vector and returns it. +func (p *Vector) Copy() Vector { + arr := make(Vector, len(*p)) + copy(arr, *p) + return arr +} + + +// Insert inserts into the vector an element of value x before +// the current element at index i. +func (p *Vector) Insert(i int, x interface{}) { + p.Expand(i, 1) + (*p)[i] = x +} + + +// Delete deletes the i'th element of the vector. The gap is closed so the old +// element at index i+1 has index i afterwards. +func (p *Vector) Delete(i int) { + a := *p + n := len(a) + + copy(a[i:n-1], a[i+1:n]) + var zero interface{} + a[n-1] = zero // support GC, zero out entry + *p = a[0 : n-1] +} + + +// InsertVector inserts into the vector the contents of the vector +// x such that the 0th element of x appears at index i after insertion. +func (p *Vector) InsertVector(i int, x *Vector) { + b := *x + + p.Expand(i, len(b)) + copy((*p)[i:i+len(b)], b) +} + + +// Cut deletes elements i through j-1, inclusive. +func (p *Vector) Cut(i, j int) { + a := *p + n := len(a) + m := n - (j - i) + + copy(a[i:m], a[j:n]) + for k := m; k < n; k++ { //TODO(bflm) don't zero out the elements unless it's a Vector. + var zero interface{} + a[k] = zero // support GC, zero out entries + } + + *p = a[0:m] +} + + +// Slice returns a new sub-vector by slicing the old one to extract slice [i:j]. +// The elements are copied. The original vector is unchanged. +func (p *Vector) Slice(i, j int) *Vector { + var s Vector + s.realloc(j-i, 0) // will fail in Init() if j < i + copy(s, (*p)[i:j]) + return &s +} + + +// Convenience wrappers + +// Push appends x to the end of the vector. +func (p *Vector) Push(x interface{}) { p.Insert(len(*p), x) } + + +// Pop deletes the last element of the vector. +func (p *Vector) Pop() interface{} { + a := *p + + i := len(a) - 1 + x := a[i] + var zero interface{} + a[i] = zero // support GC, zero out entry + *p = a[0:i] + return x +} + + +// AppendVector appends the entire vector x to the end of this vector. +func (p *Vector) AppendVector(x *Vector) { p.InsertVector(len(*p), x) } + + +// Swap exchanges the elements at indexes i and j. +func (p *Vector) Swap(i, j int) { + a := *p + a[i], a[j] = a[j], a[i] +} + + +// Do calls function f for each element of the vector, in order. +// The behavior of Do is undefined if f changes *p. +func (p *Vector) Do(f func(elem interface{})) { + for _, e := range *p { + f(e) + } +} diff --git a/libgo/go/container/vector/vector_test.go b/libgo/go/container/vector/vector_test.go new file mode 100644 index 000000000..a9c4ceb55 --- /dev/null +++ b/libgo/go/container/vector/vector_test.go @@ -0,0 +1,344 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// CAUTION: If this file is not vector_test.go, it was generated +// automatically from vector_test.go - DO NOT EDIT in that case! + +package vector + +import "testing" + + +func TestZeroLen(t *testing.T) { + a := new(Vector) + if a.Len() != 0 { + t.Errorf("%T: B1) expected 0, got %d", a, a.Len()) + } + if len(*a) != 0 { + t.Errorf("%T: B2) expected 0, got %d", a, len(*a)) + } + var b Vector + if b.Len() != 0 { + t.Errorf("%T: B3) expected 0, got %d", b, b.Len()) + } + if len(b) != 0 { + t.Errorf("%T: B4) expected 0, got %d", b, len(b)) + } +} + + +func TestResize(t *testing.T) { + var a Vector + checkSize(t, &a, 0, 0) + checkSize(t, a.Resize(0, 5), 0, 5) + checkSize(t, a.Resize(1, 0), 1, 5) + checkSize(t, a.Resize(10, 0), 10, 10) + checkSize(t, a.Resize(5, 0), 5, 10) + checkSize(t, a.Resize(3, 8), 3, 10) + checkSize(t, a.Resize(0, 100), 0, 100) + checkSize(t, a.Resize(11, 100), 11, 100) +} + + +func TestResize2(t *testing.T) { + var a Vector + checkSize(t, &a, 0, 0) + a.Push(int2Value(1)) + a.Push(int2Value(2)) + a.Push(int2Value(3)) + a.Push(int2Value(4)) + checkSize(t, &a, 4, 4) + checkSize(t, a.Resize(10, 0), 10, 10) + for i := 4; i < a.Len(); i++ { + if a.At(i) != zero { + t.Errorf("%T: expected a.At(%d) == %v; found %v!", a, i, zero, a.At(i)) + } + } + for i := 4; i < len(a); i++ { + if a[i] != zero { + t.Errorf("%T: expected a[%d] == %v; found %v", a, i, zero, a[i]) + } + } +} + + +func checkZero(t *testing.T, a *Vector, i int) { + for j := 0; j < i; j++ { + if a.At(j) == zero { + t.Errorf("%T: 1 expected a.At(%d) == %d; found %v", a, j, j, a.At(j)) + } + if (*a)[j] == zero { + t.Errorf("%T: 2 expected (*a)[%d] == %d; found %v", a, j, j, (*a)[j]) + } + } + for ; i < a.Len(); i++ { + if a.At(i) != zero { + t.Errorf("%T: 3 expected a.At(%d) == %v; found %v", a, i, zero, a.At(i)) + } + if (*a)[i] != zero { + t.Errorf("%T: 4 expected (*a)[%d] == %v; found %v", a, i, zero, (*a)[i]) + } + } +} + + +func TestTrailingElements(t *testing.T) { + var a Vector + for i := 0; i < 10; i++ { + a.Push(int2Value(i + 1)) + } + checkZero(t, &a, 10) + checkSize(t, &a, 10, 16) + checkSize(t, a.Resize(5, 0), 5, 16) + checkSize(t, a.Resize(10, 0), 10, 16) + checkZero(t, &a, 5) +} + + +func TestAccess(t *testing.T) { + const n = 100 + var a Vector + a.Resize(n, 0) + for i := 0; i < n; i++ { + a.Set(i, int2Value(val(i))) + } + for i := 0; i < n; i++ { + if elem2Value(a.At(i)) != int2Value(val(i)) { + t.Error(i) + } + } + var b Vector + b.Resize(n, 0) + for i := 0; i < n; i++ { + b[i] = int2Value(val(i)) + } + for i := 0; i < n; i++ { + if elem2Value(b[i]) != int2Value(val(i)) { + t.Error(i) + } + } +} + + +func TestInsertDeleteClear(t *testing.T) { + const n = 100 + var a Vector + + for i := 0; i < n; i++ { + if a.Len() != i { + t.Errorf("%T: A) wrong Len() %d (expected %d)", a, a.Len(), i) + } + if len(a) != i { + t.Errorf("%T: A) wrong len() %d (expected %d)", a, len(a), i) + } + a.Insert(0, int2Value(val(i))) + if elem2Value(a.Last()) != int2Value(val(0)) { + t.Errorf("%T: B", a) + } + } + for i := n - 1; i >= 0; i-- { + if elem2Value(a.Last()) != int2Value(val(0)) { + t.Errorf("%T: C", a) + } + if elem2Value(a.At(0)) != int2Value(val(i)) { + t.Errorf("%T: D", a) + } + if elem2Value(a[0]) != int2Value(val(i)) { + t.Errorf("%T: D2", a) + } + a.Delete(0) + if a.Len() != i { + t.Errorf("%T: E) wrong Len() %d (expected %d)", a, a.Len(), i) + } + if len(a) != i { + t.Errorf("%T: E) wrong len() %d (expected %d)", a, len(a), i) + } + } + + if a.Len() != 0 { + t.Errorf("%T: F) wrong Len() %d (expected 0)", a, a.Len()) + } + if len(a) != 0 { + t.Errorf("%T: F) wrong len() %d (expected 0)", a, len(a)) + } + for i := 0; i < n; i++ { + a.Push(int2Value(val(i))) + if a.Len() != i+1 { + t.Errorf("%T: G) wrong Len() %d (expected %d)", a, a.Len(), i+1) + } + if len(a) != i+1 { + t.Errorf("%T: G) wrong len() %d (expected %d)", a, len(a), i+1) + } + if elem2Value(a.Last()) != int2Value(val(i)) { + t.Errorf("%T: H", a) + } + } + a.Resize(0, 0) + if a.Len() != 0 { + t.Errorf("%T: I wrong Len() %d (expected 0)", a, a.Len()) + } + if len(a) != 0 { + t.Errorf("%T: I wrong len() %d (expected 0)", a, len(a)) + } + + const m = 5 + for j := 0; j < m; j++ { + a.Push(int2Value(j)) + for i := 0; i < n; i++ { + x := val(i) + a.Push(int2Value(x)) + if elem2Value(a.Pop()) != int2Value(x) { + t.Errorf("%T: J", a) + } + if a.Len() != j+1 { + t.Errorf("%T: K) wrong Len() %d (expected %d)", a, a.Len(), j+1) + } + if len(a) != j+1 { + t.Errorf("%T: K) wrong len() %d (expected %d)", a, len(a), j+1) + } + } + } + if a.Len() != m { + t.Errorf("%T: L) wrong Len() %d (expected %d)", a, a.Len(), m) + } + if len(a) != m { + t.Errorf("%T: L) wrong len() %d (expected %d)", a, len(a), m) + } +} + + +func verify_slice(t *testing.T, x *Vector, elt, i, j int) { + for k := i; k < j; k++ { + if elem2Value(x.At(k)) != int2Value(elt) { + t.Errorf("%T: M) wrong [%d] element %v (expected %v)", x, k, elem2Value(x.At(k)), int2Value(elt)) + } + } + + s := x.Slice(i, j) + for k, n := 0, j-i; k < n; k++ { + if elem2Value(s.At(k)) != int2Value(elt) { + t.Errorf("%T: N) wrong [%d] element %v (expected %v)", x, k, elem2Value(x.At(k)), int2Value(elt)) + } + } +} + + +func verify_pattern(t *testing.T, x *Vector, a, b, c int) { + n := a + b + c + if x.Len() != n { + t.Errorf("%T: O) wrong Len() %d (expected %d)", x, x.Len(), n) + } + if len(*x) != n { + t.Errorf("%T: O) wrong len() %d (expected %d)", x, len(*x), n) + } + verify_slice(t, x, 0, 0, a) + verify_slice(t, x, 1, a, a+b) + verify_slice(t, x, 0, a+b, n) +} + + +func make_vector(elt, len int) *Vector { + x := new(Vector).Resize(len, 0) + for i := 0; i < len; i++ { + x.Set(i, int2Value(elt)) + } + return x +} + + +func TestInsertVector(t *testing.T) { + // 1 + a := make_vector(0, 0) + b := make_vector(1, 10) + a.InsertVector(0, b) + verify_pattern(t, a, 0, 10, 0) + // 2 + a = make_vector(0, 10) + b = make_vector(1, 0) + a.InsertVector(5, b) + verify_pattern(t, a, 5, 0, 5) + // 3 + a = make_vector(0, 10) + b = make_vector(1, 3) + a.InsertVector(3, b) + verify_pattern(t, a, 3, 3, 7) + // 4 + a = make_vector(0, 10) + b = make_vector(1, 1000) + a.InsertVector(8, b) + verify_pattern(t, a, 8, 1000, 2) +} + + +func TestDo(t *testing.T) { + const n = 25 + const salt = 17 + a := new(Vector).Resize(n, 0) + for i := 0; i < n; i++ { + a.Set(i, int2Value(salt*i)) + } + count := 0 + a.Do(func(e interface{}) { + i := intf2Value(e) + if i != int2Value(count*salt) { + t.Error(tname(a), "value at", count, "should be", count*salt, "not", i) + } + count++ + }) + if count != n { + t.Error(tname(a), "should visit", n, "values; did visit", count) + } + + b := new(Vector).Resize(n, 0) + for i := 0; i < n; i++ { + (*b)[i] = int2Value(salt * i) + } + count = 0 + b.Do(func(e interface{}) { + i := intf2Value(e) + if i != int2Value(count*salt) { + t.Error(tname(b), "b) value at", count, "should be", count*salt, "not", i) + } + count++ + }) + if count != n { + t.Error(tname(b), "b) should visit", n, "values; did visit", count) + } + + var c Vector + c.Resize(n, 0) + for i := 0; i < n; i++ { + c[i] = int2Value(salt * i) + } + count = 0 + c.Do(func(e interface{}) { + i := intf2Value(e) + if i != int2Value(count*salt) { + t.Error(tname(c), "c) value at", count, "should be", count*salt, "not", i) + } + count++ + }) + if count != n { + t.Error(tname(c), "c) should visit", n, "values; did visit", count) + } + +} + + +func TestVectorCopy(t *testing.T) { + // verify Copy() returns a copy, not simply a slice of the original vector + const Len = 10 + var src Vector + for i := 0; i < Len; i++ { + src.Push(int2Value(i * i)) + } + dest := src.Copy() + for i := 0; i < Len; i++ { + src[i] = int2Value(-1) + v := elem2Value(dest[i]) + if v != int2Value(i*i) { + t.Error(tname(src), "expected", i*i, "got", v) + } + } +} diff --git a/libgo/go/crypto/aes/aes_test.go b/libgo/go/crypto/aes/aes_test.go new file mode 100644 index 000000000..2136d447d --- /dev/null +++ b/libgo/go/crypto/aes/aes_test.go @@ -0,0 +1,350 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package aes + +import ( + "testing" +) + +// See const.go for overview of math here. + +// Test that powx is initialized correctly. +// (Can adapt this code to generate it too.) +func TestPowx(t *testing.T) { + p := 1 + for i := 0; i < len(powx); i++ { + if powx[i] != byte(p) { + t.Errorf("powx[%d] = %#x, want %#x", i, powx[i], p) + } + p <<= 1 + if p&0x100 != 0 { + p ^= poly + } + } +} + +// Multiply b and c as GF(2) polynomials modulo poly +func mul(b, c uint32) uint32 { + i := b + j := c + s := uint32(0) + for k := uint32(1); k < 0x100 && j != 0; k <<= 1 { + // Invariant: k == 1<>8 + } + } +} + +// Test that decryption tables are correct. +// (Can adapt this code to generate them too.) +func TestTd(t *testing.T) { + for i := 0; i < 256; i++ { + s := uint32(sbox1[i]) + s9 := mul(s, 0x9) + sb := mul(s, 0xb) + sd := mul(s, 0xd) + se := mul(s, 0xe) + w := se<<24 | s9<<16 | sd<<8 | sb + for j := 0; j < 4; j++ { + if x := td[j][i]; x != w { + t.Fatalf("td[%d][%d] = %#x, want %#x", j, i, x, w) + } + w = w<<24 | w>>8 + } + } +} + +// Test vectors are from FIPS 197: +// http://www.csrc.nist.gov/publications/fips/fips197/fips-197.pdf + +// Appendix A of FIPS 197: Key expansion examples +type KeyTest struct { + key []byte + enc []uint32 + dec []uint32 // decryption expansion; not in FIPS 197, computed from C implementation. +} + +var keyTests = []KeyTest{ + { + // A.1. Expansion of a 128-bit Cipher Key + []byte{0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c}, + []uint32{ + 0x2b7e1516, 0x28aed2a6, 0xabf71588, 0x09cf4f3c, + 0xa0fafe17, 0x88542cb1, 0x23a33939, 0x2a6c7605, + 0xf2c295f2, 0x7a96b943, 0x5935807a, 0x7359f67f, + 0x3d80477d, 0x4716fe3e, 0x1e237e44, 0x6d7a883b, + 0xef44a541, 0xa8525b7f, 0xb671253b, 0xdb0bad00, + 0xd4d1c6f8, 0x7c839d87, 0xcaf2b8bc, 0x11f915bc, + 0x6d88a37a, 0x110b3efd, 0xdbf98641, 0xca0093fd, + 0x4e54f70e, 0x5f5fc9f3, 0x84a64fb2, 0x4ea6dc4f, + 0xead27321, 0xb58dbad2, 0x312bf560, 0x7f8d292f, + 0xac7766f3, 0x19fadc21, 0x28d12941, 0x575c006e, + 0xd014f9a8, 0xc9ee2589, 0xe13f0cc8, 0xb6630ca6, + }, + []uint32{ + 0xd014f9a8, 0xc9ee2589, 0xe13f0cc8, 0xb6630ca6, + 0xc7b5a63, 0x1319eafe, 0xb0398890, 0x664cfbb4, + 0xdf7d925a, 0x1f62b09d, 0xa320626e, 0xd6757324, + 0x12c07647, 0xc01f22c7, 0xbc42d2f3, 0x7555114a, + 0x6efcd876, 0xd2df5480, 0x7c5df034, 0xc917c3b9, + 0x6ea30afc, 0xbc238cf6, 0xae82a4b4, 0xb54a338d, + 0x90884413, 0xd280860a, 0x12a12842, 0x1bc89739, + 0x7c1f13f7, 0x4208c219, 0xc021ae48, 0x969bf7b, + 0xcc7505eb, 0x3e17d1ee, 0x82296c51, 0xc9481133, + 0x2b3708a7, 0xf262d405, 0xbc3ebdbf, 0x4b617d62, + 0x2b7e1516, 0x28aed2a6, 0xabf71588, 0x9cf4f3c, + }, + }, + { + // A.2. Expansion of a 192-bit Cipher Key + []byte{ + 0x8e, 0x73, 0xb0, 0xf7, 0xda, 0x0e, 0x64, 0x52, 0xc8, 0x10, 0xf3, 0x2b, 0x80, 0x90, 0x79, 0xe5, + 0x62, 0xf8, 0xea, 0xd2, 0x52, 0x2c, 0x6b, 0x7b, + }, + []uint32{ + 0x8e73b0f7, 0xda0e6452, 0xc810f32b, 0x809079e5, + 0x62f8ead2, 0x522c6b7b, 0xfe0c91f7, 0x2402f5a5, + 0xec12068e, 0x6c827f6b, 0x0e7a95b9, 0x5c56fec2, + 0x4db7b4bd, 0x69b54118, 0x85a74796, 0xe92538fd, + 0xe75fad44, 0xbb095386, 0x485af057, 0x21efb14f, + 0xa448f6d9, 0x4d6dce24, 0xaa326360, 0x113b30e6, + 0xa25e7ed5, 0x83b1cf9a, 0x27f93943, 0x6a94f767, + 0xc0a69407, 0xd19da4e1, 0xec1786eb, 0x6fa64971, + 0x485f7032, 0x22cb8755, 0xe26d1352, 0x33f0b7b3, + 0x40beeb28, 0x2f18a259, 0x6747d26b, 0x458c553e, + 0xa7e1466c, 0x9411f1df, 0x821f750a, 0xad07d753, + 0xca400538, 0x8fcc5006, 0x282d166a, 0xbc3ce7b5, + 0xe98ba06f, 0x448c773c, 0x8ecc7204, 0x01002202, + }, + nil, + }, + { + // A.3. Expansion of a 256-bit Cipher Key + []byte{ + 0x60, 0x3d, 0xeb, 0x10, 0x15, 0xca, 0x71, 0xbe, 0x2b, 0x73, 0xae, 0xf0, 0x85, 0x7d, 0x77, 0x81, + 0x1f, 0x35, 0x2c, 0x07, 0x3b, 0x61, 0x08, 0xd7, 0x2d, 0x98, 0x10, 0xa3, 0x09, 0x14, 0xdf, 0xf4, + }, + []uint32{ + 0x603deb10, 0x15ca71be, 0x2b73aef0, 0x857d7781, + 0x1f352c07, 0x3b6108d7, 0x2d9810a3, 0x0914dff4, + 0x9ba35411, 0x8e6925af, 0xa51a8b5f, 0x2067fcde, + 0xa8b09c1a, 0x93d194cd, 0xbe49846e, 0xb75d5b9a, + 0xd59aecb8, 0x5bf3c917, 0xfee94248, 0xde8ebe96, + 0xb5a9328a, 0x2678a647, 0x98312229, 0x2f6c79b3, + 0x812c81ad, 0xdadf48ba, 0x24360af2, 0xfab8b464, + 0x98c5bfc9, 0xbebd198e, 0x268c3ba7, 0x09e04214, + 0x68007bac, 0xb2df3316, 0x96e939e4, 0x6c518d80, + 0xc814e204, 0x76a9fb8a, 0x5025c02d, 0x59c58239, + 0xde136967, 0x6ccc5a71, 0xfa256395, 0x9674ee15, + 0x5886ca5d, 0x2e2f31d7, 0x7e0af1fa, 0x27cf73c3, + 0x749c47ab, 0x18501dda, 0xe2757e4f, 0x7401905a, + 0xcafaaae3, 0xe4d59b34, 0x9adf6ace, 0xbd10190d, + 0xfe4890d1, 0xe6188d0b, 0x046df344, 0x706c631e, + }, + nil, + }, +} + +// Test key expansion against FIPS 197 examples. +func TestExpandKey(t *testing.T) { +L: + for i, tt := range keyTests { + enc := make([]uint32, len(tt.enc)) + var dec []uint32 + if tt.dec != nil { + dec = make([]uint32, len(tt.dec)) + } + expandKey(tt.key, enc, dec) + for j, v := range enc { + if v != tt.enc[j] { + t.Errorf("key %d: enc[%d] = %#x, want %#x", i, j, v, tt.enc[j]) + continue L + } + } + if dec != nil { + for j, v := range dec { + if v != tt.dec[j] { + t.Errorf("key %d: dec[%d] = %#x, want %#x", i, j, v, tt.dec[j]) + continue L + } + } + } + } +} + +// Appendix B, C of FIPS 197: Cipher examples, Example vectors. +type CryptTest struct { + key []byte + in []byte + out []byte +} + +var encryptTests = []CryptTest{ + { + // Appendix B. + []byte{0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c}, + []byte{0x32, 0x43, 0xf6, 0xa8, 0x88, 0x5a, 0x30, 0x8d, 0x31, 0x31, 0x98, 0xa2, 0xe0, 0x37, 0x07, 0x34}, + []byte{0x39, 0x25, 0x84, 0x1d, 0x02, 0xdc, 0x09, 0xfb, 0xdc, 0x11, 0x85, 0x97, 0x19, 0x6a, 0x0b, 0x32}, + }, + { + // Appendix C.1. AES-128 + []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f}, + []byte{0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff}, + []byte{0x69, 0xc4, 0xe0, 0xd8, 0x6a, 0x7b, 0x04, 0x30, 0xd8, 0xcd, 0xb7, 0x80, 0x70, 0xb4, 0xc5, 0x5a}, + }, + { + // Appendix C.2. AES-192 + []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + }, + []byte{0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff}, + []byte{0xdd, 0xa9, 0x7c, 0xa4, 0x86, 0x4c, 0xdf, 0xe0, 0x6e, 0xaf, 0x70, 0xa0, 0xec, 0x0d, 0x71, 0x91}, + }, + { + // Appendix C.3. AES-256 + []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + }, + []byte{0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff}, + []byte{0x8e, 0xa2, 0xb7, 0xca, 0x51, 0x67, 0x45, 0xbf, 0xea, 0xfc, 0x49, 0x90, 0x4b, 0x49, 0x60, 0x89}, + }, +} + +// Test encryptBlock against FIPS 197 examples. +func TestEncryptBlock(t *testing.T) { + for i, tt := range encryptTests { + n := len(tt.key) + 28 + enc := make([]uint32, n) + dec := make([]uint32, n) + expandKey(tt.key, enc, dec) + out := make([]byte, len(tt.in)) + encryptBlock(enc, out, tt.in) + for j, v := range out { + if v != tt.out[j] { + t.Errorf("encryptBlock %d: out[%d] = %#x, want %#x", i, j, v, tt.out[j]) + break + } + } + } +} + +// Test decryptBlock against FIPS 197 examples. +func TestDecryptBlock(t *testing.T) { + for i, tt := range encryptTests { + n := len(tt.key) + 28 + enc := make([]uint32, n) + dec := make([]uint32, n) + expandKey(tt.key, enc, dec) + plain := make([]byte, len(tt.in)) + decryptBlock(dec, plain, tt.out) + for j, v := range plain { + if v != tt.in[j] { + t.Errorf("decryptBlock %d: plain[%d] = %#x, want %#x", i, j, v, tt.in[j]) + break + } + } + } +} + +// Test Cipher Encrypt method against FIPS 197 examples. +func TestCipherEncrypt(t *testing.T) { + for i, tt := range encryptTests { + c, err := NewCipher(tt.key) + if err != nil { + t.Errorf("NewCipher(%d bytes) = %s", len(tt.key), err) + continue + } + out := make([]byte, len(tt.in)) + c.Encrypt(out, tt.in) + for j, v := range out { + if v != tt.out[j] { + t.Errorf("Cipher.Encrypt %d: out[%d] = %#x, want %#x", i, j, v, tt.out[j]) + break + } + } + } +} + +// Test Cipher Decrypt against FIPS 197 examples. +func TestCipherDecrypt(t *testing.T) { + for i, tt := range encryptTests { + c, err := NewCipher(tt.key) + if err != nil { + t.Errorf("NewCipher(%d bytes) = %s", len(tt.key), err) + continue + } + plain := make([]byte, len(tt.in)) + c.Decrypt(plain, tt.out) + for j, v := range plain { + if v != tt.in[j] { + t.Errorf("decryptBlock %d: plain[%d] = %#x, want %#x", i, j, v, tt.in[j]) + break + } + } + } +} diff --git a/libgo/go/crypto/aes/block.go b/libgo/go/crypto/aes/block.go new file mode 100644 index 000000000..130cd011c --- /dev/null +++ b/libgo/go/crypto/aes/block.go @@ -0,0 +1,176 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This Go implementation is derived in part from the reference +// ANSI C implementation, which carries the following notice: +// +// rijndael-alg-fst.c +// +// @version 3.0 (December 2000) +// +// Optimised ANSI C code for the Rijndael cipher (now AES) +// +// @author Vincent Rijmen +// @author Antoon Bosselaers +// @author Paulo Barreto +// +// This code is hereby placed in the public domain. +// +// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ''AS IS'' AND ANY EXPRESS +// OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +// BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +// OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +// EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// See FIPS 197 for specification, and see Daemen and Rijmen's Rijndael submission +// for implementation details. +// http://www.csrc.nist.gov/publications/fips/fips197/fips-197.pdf +// http://csrc.nist.gov/archive/aes/rijndael/Rijndael-ammended.pdf + +package aes + +// Encrypt one block from src into dst, using the expanded key xk. +func encryptBlock(xk []uint32, dst, src []byte) { + var s0, s1, s2, s3, t0, t1, t2, t3 uint32 + + s0 = uint32(src[0])<<24 | uint32(src[1])<<16 | uint32(src[2])<<8 | uint32(src[3]) + s1 = uint32(src[4])<<24 | uint32(src[5])<<16 | uint32(src[6])<<8 | uint32(src[7]) + s2 = uint32(src[8])<<24 | uint32(src[9])<<16 | uint32(src[10])<<8 | uint32(src[11]) + s3 = uint32(src[12])<<24 | uint32(src[13])<<16 | uint32(src[14])<<8 | uint32(src[15]) + + // First round just XORs input with key. + s0 ^= xk[0] + s1 ^= xk[1] + s2 ^= xk[2] + s3 ^= xk[3] + + // Middle rounds shuffle using tables. + // Number of rounds is set by length of expanded key. + nr := len(xk)/4 - 2 // - 2: one above, one more below + k := 4 + for r := 0; r < nr; r++ { + t0 = xk[k+0] ^ te[0][s0>>24] ^ te[1][s1>>16&0xff] ^ te[2][s2>>8&0xff] ^ te[3][s3&0xff] + t1 = xk[k+1] ^ te[0][s1>>24] ^ te[1][s2>>16&0xff] ^ te[2][s3>>8&0xff] ^ te[3][s0&0xff] + t2 = xk[k+2] ^ te[0][s2>>24] ^ te[1][s3>>16&0xff] ^ te[2][s0>>8&0xff] ^ te[3][s1&0xff] + t3 = xk[k+3] ^ te[0][s3>>24] ^ te[1][s0>>16&0xff] ^ te[2][s1>>8&0xff] ^ te[3][s2&0xff] + k += 4 + s0, s1, s2, s3 = t0, t1, t2, t3 + } + + // Last round uses s-box directly and XORs to produce output. + s0 = uint32(sbox0[t0>>24])<<24 | uint32(sbox0[t1>>16&0xff])<<16 | uint32(sbox0[t2>>8&0xff])<<8 | uint32(sbox0[t3&0xff]) + s1 = uint32(sbox0[t1>>24])<<24 | uint32(sbox0[t2>>16&0xff])<<16 | uint32(sbox0[t3>>8&0xff])<<8 | uint32(sbox0[t0&0xff]) + s2 = uint32(sbox0[t2>>24])<<24 | uint32(sbox0[t3>>16&0xff])<<16 | uint32(sbox0[t0>>8&0xff])<<8 | uint32(sbox0[t1&0xff]) + s3 = uint32(sbox0[t3>>24])<<24 | uint32(sbox0[t0>>16&0xff])<<16 | uint32(sbox0[t1>>8&0xff])<<8 | uint32(sbox0[t2&0xff]) + + s0 ^= xk[k+0] + s1 ^= xk[k+1] + s2 ^= xk[k+2] + s3 ^= xk[k+3] + + dst[0], dst[1], dst[2], dst[3] = byte(s0>>24), byte(s0>>16), byte(s0>>8), byte(s0) + dst[4], dst[5], dst[6], dst[7] = byte(s1>>24), byte(s1>>16), byte(s1>>8), byte(s1) + dst[8], dst[9], dst[10], dst[11] = byte(s2>>24), byte(s2>>16), byte(s2>>8), byte(s2) + dst[12], dst[13], dst[14], dst[15] = byte(s3>>24), byte(s3>>16), byte(s3>>8), byte(s3) +} + +// Decrypt one block from src into dst, using the expanded key xk. +func decryptBlock(xk []uint32, dst, src []byte) { + var s0, s1, s2, s3, t0, t1, t2, t3 uint32 + + s0 = uint32(src[0])<<24 | uint32(src[1])<<16 | uint32(src[2])<<8 | uint32(src[3]) + s1 = uint32(src[4])<<24 | uint32(src[5])<<16 | uint32(src[6])<<8 | uint32(src[7]) + s2 = uint32(src[8])<<24 | uint32(src[9])<<16 | uint32(src[10])<<8 | uint32(src[11]) + s3 = uint32(src[12])<<24 | uint32(src[13])<<16 | uint32(src[14])<<8 | uint32(src[15]) + + // First round just XORs input with key. + s0 ^= xk[0] + s1 ^= xk[1] + s2 ^= xk[2] + s3 ^= xk[3] + + // Middle rounds shuffle using tables. + // Number of rounds is set by length of expanded key. + nr := len(xk)/4 - 2 // - 2: one above, one more below + k := 4 + for r := 0; r < nr; r++ { + t0 = xk[k+0] ^ td[0][s0>>24] ^ td[1][s3>>16&0xff] ^ td[2][s2>>8&0xff] ^ td[3][s1&0xff] + t1 = xk[k+1] ^ td[0][s1>>24] ^ td[1][s0>>16&0xff] ^ td[2][s3>>8&0xff] ^ td[3][s2&0xff] + t2 = xk[k+2] ^ td[0][s2>>24] ^ td[1][s1>>16&0xff] ^ td[2][s0>>8&0xff] ^ td[3][s3&0xff] + t3 = xk[k+3] ^ td[0][s3>>24] ^ td[1][s2>>16&0xff] ^ td[2][s1>>8&0xff] ^ td[3][s0&0xff] + k += 4 + s0, s1, s2, s3 = t0, t1, t2, t3 + } + + // Last round uses s-box directly and XORs to produce output. + s0 = uint32(sbox1[t0>>24])<<24 | uint32(sbox1[t3>>16&0xff])<<16 | uint32(sbox1[t2>>8&0xff])<<8 | uint32(sbox1[t1&0xff]) + s1 = uint32(sbox1[t1>>24])<<24 | uint32(sbox1[t0>>16&0xff])<<16 | uint32(sbox1[t3>>8&0xff])<<8 | uint32(sbox1[t2&0xff]) + s2 = uint32(sbox1[t2>>24])<<24 | uint32(sbox1[t1>>16&0xff])<<16 | uint32(sbox1[t0>>8&0xff])<<8 | uint32(sbox1[t3&0xff]) + s3 = uint32(sbox1[t3>>24])<<24 | uint32(sbox1[t2>>16&0xff])<<16 | uint32(sbox1[t1>>8&0xff])<<8 | uint32(sbox1[t0&0xff]) + + s0 ^= xk[k+0] + s1 ^= xk[k+1] + s2 ^= xk[k+2] + s3 ^= xk[k+3] + + dst[0], dst[1], dst[2], dst[3] = byte(s0>>24), byte(s0>>16), byte(s0>>8), byte(s0) + dst[4], dst[5], dst[6], dst[7] = byte(s1>>24), byte(s1>>16), byte(s1>>8), byte(s1) + dst[8], dst[9], dst[10], dst[11] = byte(s2>>24), byte(s2>>16), byte(s2>>8), byte(s2) + dst[12], dst[13], dst[14], dst[15] = byte(s3>>24), byte(s3>>16), byte(s3>>8), byte(s3) +} + +// Apply sbox0 to each byte in w. +func subw(w uint32) uint32 { + return uint32(sbox0[w>>24])<<24 | + uint32(sbox0[w>>16&0xff])<<16 | + uint32(sbox0[w>>8&0xff])<<8 | + uint32(sbox0[w&0xff]) +} + +// Rotate +func rotw(w uint32) uint32 { return w<<8 | w>>24 } + +// Key expansion algorithm. See FIPS-197, Figure 11. +// Their rcon[i] is our powx[i-1] << 24. +func expandKey(key []byte, enc, dec []uint32) { + // Encryption key setup. + var i int + nk := len(key) / 4 + for i = 0; i < nk; i++ { + enc[i] = uint32(key[4*i])<<24 | uint32(key[4*i+1])<<16 | uint32(key[4*i+2])<<8 | uint32(key[4*i+3]) + } + for ; i < len(enc); i++ { + t := enc[i-1] + if i%nk == 0 { + t = subw(rotw(t)) ^ (uint32(powx[i/nk-1]) << 24) + } else if nk > 6 && i%nk == 4 { + t = subw(t) + } + enc[i] = enc[i-nk] ^ t + } + + // Derive decryption key from encryption key. + // Reverse the 4-word round key sets from enc to produce dec. + // All sets but the first and last get the MixColumn transform applied. + if dec == nil { + return + } + n := len(enc) + for i := 0; i < n; i += 4 { + ei := n - i - 4 + for j := 0; j < 4; j++ { + x := enc[ei+j] + if i > 0 && i+4 < n { + x = td[0][sbox0[x>>24]] ^ td[1][sbox0[x>>16&0xff]] ^ td[2][sbox0[x>>8&0xff]] ^ td[3][sbox0[x&0xff]] + } + dec[i+j] = x + } + } +} diff --git a/libgo/go/crypto/aes/cipher.go b/libgo/go/crypto/aes/cipher.go new file mode 100644 index 000000000..3a9d02318 --- /dev/null +++ b/libgo/go/crypto/aes/cipher.go @@ -0,0 +1,71 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package aes + +import ( + "os" + "strconv" +) + +// The AES block size in bytes. +const BlockSize = 16 + +// A Cipher is an instance of AES encryption using a particular key. +type Cipher struct { + enc []uint32 + dec []uint32 +} + +type KeySizeError int + +func (k KeySizeError) String() string { + return "crypto/aes: invalid key size " + strconv.Itoa(int(k)) +} + +// NewCipher creates and returns a new Cipher. +// The key argument should be the AES key, +// either 16, 24, or 32 bytes to select +// AES-128, AES-192, or AES-256. +func NewCipher(key []byte) (*Cipher, os.Error) { + k := len(key) + switch k { + default: + return nil, KeySizeError(k) + case 16, 24, 32: + break + } + + n := k + 28 + c := &Cipher{make([]uint32, n), make([]uint32, n)} + expandKey(key, c.enc, c.dec) + return c, nil +} + +// BlockSize returns the AES block size, 16 bytes. +// It is necessary to satisfy the Cipher interface in the +// package "crypto/block". +func (c *Cipher) BlockSize() int { return BlockSize } + +// Encrypt encrypts the 16-byte buffer src using the key k +// and stores the result in dst. +// Note that for amounts of data larger than a block, +// it is not safe to just call Encrypt on successive blocks; +// instead, use an encryption mode like CBC (see crypto/block/cbc.go). +func (c *Cipher) Encrypt(dst, src []byte) { encryptBlock(c.enc, dst, src) } + +// Decrypt decrypts the 16-byte buffer src using the key k +// and stores the result in dst. +func (c *Cipher) Decrypt(dst, src []byte) { decryptBlock(c.dec, dst, src) } + +// Reset zeros the key data, so that it will no longer +// appear in the process's memory. +func (c *Cipher) Reset() { + for i := 0; i < len(c.enc); i++ { + c.enc[i] = 0 + } + for i := 0; i < len(c.dec); i++ { + c.dec[i] = 0 + } +} diff --git a/libgo/go/crypto/aes/const.go b/libgo/go/crypto/aes/const.go new file mode 100644 index 000000000..97a5b64ec --- /dev/null +++ b/libgo/go/crypto/aes/const.go @@ -0,0 +1,362 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// AES constants - 8720 bytes of initialized data. + +// This package implements AES encryption (formerly Rijndael), +// as defined in U.S. Federal Information Processing Standards Publication 197. +package aes + +// http://www.csrc.nist.gov/publications/fips/fips197/fips-197.pdf + +// AES is based on the mathematical behavior of binary polynomials +// (polynomials over GF(2)) modulo the irreducible polynomial x⁸ + x⁴ + x² + x + 1. +// Addition of these binary polynomials corresponds to binary xor. +// Reducing mod poly corresponds to binary xor with poly every +// time a 0x100 bit appears. +const poly = 1<<8 | 1<<4 | 1<<3 | 1<<1 | 1<<0 // x⁸ + x⁴ + x² + x + 1 + +// Powers of x mod poly in GF(2). +var powx = [16]byte{ + 0x01, + 0x02, + 0x04, + 0x08, + 0x10, + 0x20, + 0x40, + 0x80, + 0x1b, + 0x36, + 0x6c, + 0xd8, + 0xab, + 0x4d, + 0x9a, + 0x2f, +} + +// FIPS-197 Figure 7. S-box substitution values in hexadecimal format. +var sbox0 = [256]byte{ + 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76, + 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0, + 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc, 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15, + 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75, + 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0, 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84, + 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b, 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf, + 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8, + 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5, 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2, + 0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17, 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73, + 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb, + 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c, 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79, + 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9, 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08, + 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a, + 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e, + 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94, 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf, + 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16, +} + +// FIPS-197 Figure 14. Inverse S-box substitution values in hexadecimal format. +var sbox1 = [256]byte{ + 0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38, 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb, + 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87, 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb, + 0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d, 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e, + 0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2, 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25, + 0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16, 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92, + 0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda, 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84, + 0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a, 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06, + 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02, 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b, + 0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea, 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73, + 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85, 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e, + 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89, 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b, + 0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20, 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4, + 0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31, 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f, + 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d, 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef, + 0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0, 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61, + 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26, 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d, +} + +// Lookup tables for encryption. +// These can be recomputed by adapting the tests in aes_test.go. + +var te = [4][256]uint32{ + { + 0xc66363a5, 0xf87c7c84, 0xee777799, 0xf67b7b8d, 0xfff2f20d, 0xd66b6bbd, 0xde6f6fb1, 0x91c5c554, + 0x60303050, 0x02010103, 0xce6767a9, 0x562b2b7d, 0xe7fefe19, 0xb5d7d762, 0x4dababe6, 0xec76769a, + 0x8fcaca45, 0x1f82829d, 0x89c9c940, 0xfa7d7d87, 0xeffafa15, 0xb25959eb, 0x8e4747c9, 0xfbf0f00b, + 0x41adadec, 0xb3d4d467, 0x5fa2a2fd, 0x45afafea, 0x239c9cbf, 0x53a4a4f7, 0xe4727296, 0x9bc0c05b, + 0x75b7b7c2, 0xe1fdfd1c, 0x3d9393ae, 0x4c26266a, 0x6c36365a, 0x7e3f3f41, 0xf5f7f702, 0x83cccc4f, + 0x6834345c, 0x51a5a5f4, 0xd1e5e534, 0xf9f1f108, 0xe2717193, 0xabd8d873, 0x62313153, 0x2a15153f, + 0x0804040c, 0x95c7c752, 0x46232365, 0x9dc3c35e, 0x30181828, 0x379696a1, 0x0a05050f, 0x2f9a9ab5, + 0x0e070709, 0x24121236, 0x1b80809b, 0xdfe2e23d, 0xcdebeb26, 0x4e272769, 0x7fb2b2cd, 0xea75759f, + 0x1209091b, 0x1d83839e, 0x582c2c74, 0x341a1a2e, 0x361b1b2d, 0xdc6e6eb2, 0xb45a5aee, 0x5ba0a0fb, + 0xa45252f6, 0x763b3b4d, 0xb7d6d661, 0x7db3b3ce, 0x5229297b, 0xdde3e33e, 0x5e2f2f71, 0x13848497, + 0xa65353f5, 0xb9d1d168, 0x00000000, 0xc1eded2c, 0x40202060, 0xe3fcfc1f, 0x79b1b1c8, 0xb65b5bed, + 0xd46a6abe, 0x8dcbcb46, 0x67bebed9, 0x7239394b, 0x944a4ade, 0x984c4cd4, 0xb05858e8, 0x85cfcf4a, + 0xbbd0d06b, 0xc5efef2a, 0x4faaaae5, 0xedfbfb16, 0x864343c5, 0x9a4d4dd7, 0x66333355, 0x11858594, + 0x8a4545cf, 0xe9f9f910, 0x04020206, 0xfe7f7f81, 0xa05050f0, 0x783c3c44, 0x259f9fba, 0x4ba8a8e3, + 0xa25151f3, 0x5da3a3fe, 0x804040c0, 0x058f8f8a, 0x3f9292ad, 0x219d9dbc, 0x70383848, 0xf1f5f504, + 0x63bcbcdf, 0x77b6b6c1, 0xafdada75, 0x42212163, 0x20101030, 0xe5ffff1a, 0xfdf3f30e, 0xbfd2d26d, + 0x81cdcd4c, 0x180c0c14, 0x26131335, 0xc3ecec2f, 0xbe5f5fe1, 0x359797a2, 0x884444cc, 0x2e171739, + 0x93c4c457, 0x55a7a7f2, 0xfc7e7e82, 0x7a3d3d47, 0xc86464ac, 0xba5d5de7, 0x3219192b, 0xe6737395, + 0xc06060a0, 0x19818198, 0x9e4f4fd1, 0xa3dcdc7f, 0x44222266, 0x542a2a7e, 0x3b9090ab, 0x0b888883, + 0x8c4646ca, 0xc7eeee29, 0x6bb8b8d3, 0x2814143c, 0xa7dede79, 0xbc5e5ee2, 0x160b0b1d, 0xaddbdb76, + 0xdbe0e03b, 0x64323256, 0x743a3a4e, 0x140a0a1e, 0x924949db, 0x0c06060a, 0x4824246c, 0xb85c5ce4, + 0x9fc2c25d, 0xbdd3d36e, 0x43acacef, 0xc46262a6, 0x399191a8, 0x319595a4, 0xd3e4e437, 0xf279798b, + 0xd5e7e732, 0x8bc8c843, 0x6e373759, 0xda6d6db7, 0x018d8d8c, 0xb1d5d564, 0x9c4e4ed2, 0x49a9a9e0, + 0xd86c6cb4, 0xac5656fa, 0xf3f4f407, 0xcfeaea25, 0xca6565af, 0xf47a7a8e, 0x47aeaee9, 0x10080818, + 0x6fbabad5, 0xf0787888, 0x4a25256f, 0x5c2e2e72, 0x381c1c24, 0x57a6a6f1, 0x73b4b4c7, 0x97c6c651, + 0xcbe8e823, 0xa1dddd7c, 0xe874749c, 0x3e1f1f21, 0x964b4bdd, 0x61bdbddc, 0x0d8b8b86, 0x0f8a8a85, + 0xe0707090, 0x7c3e3e42, 0x71b5b5c4, 0xcc6666aa, 0x904848d8, 0x06030305, 0xf7f6f601, 0x1c0e0e12, + 0xc26161a3, 0x6a35355f, 0xae5757f9, 0x69b9b9d0, 0x17868691, 0x99c1c158, 0x3a1d1d27, 0x279e9eb9, + 0xd9e1e138, 0xebf8f813, 0x2b9898b3, 0x22111133, 0xd26969bb, 0xa9d9d970, 0x078e8e89, 0x339494a7, + 0x2d9b9bb6, 0x3c1e1e22, 0x15878792, 0xc9e9e920, 0x87cece49, 0xaa5555ff, 0x50282878, 0xa5dfdf7a, + 0x038c8c8f, 0x59a1a1f8, 0x09898980, 0x1a0d0d17, 0x65bfbfda, 0xd7e6e631, 0x844242c6, 0xd06868b8, + 0x824141c3, 0x299999b0, 0x5a2d2d77, 0x1e0f0f11, 0x7bb0b0cb, 0xa85454fc, 0x6dbbbbd6, 0x2c16163a, + }, + { + 0xa5c66363, 0x84f87c7c, 0x99ee7777, 0x8df67b7b, 0x0dfff2f2, 0xbdd66b6b, 0xb1de6f6f, 0x5491c5c5, + 0x50603030, 0x03020101, 0xa9ce6767, 0x7d562b2b, 0x19e7fefe, 0x62b5d7d7, 0xe64dabab, 0x9aec7676, + 0x458fcaca, 0x9d1f8282, 0x4089c9c9, 0x87fa7d7d, 0x15effafa, 0xebb25959, 0xc98e4747, 0x0bfbf0f0, + 0xec41adad, 0x67b3d4d4, 0xfd5fa2a2, 0xea45afaf, 0xbf239c9c, 0xf753a4a4, 0x96e47272, 0x5b9bc0c0, + 0xc275b7b7, 0x1ce1fdfd, 0xae3d9393, 0x6a4c2626, 0x5a6c3636, 0x417e3f3f, 0x02f5f7f7, 0x4f83cccc, + 0x5c683434, 0xf451a5a5, 0x34d1e5e5, 0x08f9f1f1, 0x93e27171, 0x73abd8d8, 0x53623131, 0x3f2a1515, + 0x0c080404, 0x5295c7c7, 0x65462323, 0x5e9dc3c3, 0x28301818, 0xa1379696, 0x0f0a0505, 0xb52f9a9a, + 0x090e0707, 0x36241212, 0x9b1b8080, 0x3ddfe2e2, 0x26cdebeb, 0x694e2727, 0xcd7fb2b2, 0x9fea7575, + 0x1b120909, 0x9e1d8383, 0x74582c2c, 0x2e341a1a, 0x2d361b1b, 0xb2dc6e6e, 0xeeb45a5a, 0xfb5ba0a0, + 0xf6a45252, 0x4d763b3b, 0x61b7d6d6, 0xce7db3b3, 0x7b522929, 0x3edde3e3, 0x715e2f2f, 0x97138484, + 0xf5a65353, 0x68b9d1d1, 0x00000000, 0x2cc1eded, 0x60402020, 0x1fe3fcfc, 0xc879b1b1, 0xedb65b5b, + 0xbed46a6a, 0x468dcbcb, 0xd967bebe, 0x4b723939, 0xde944a4a, 0xd4984c4c, 0xe8b05858, 0x4a85cfcf, + 0x6bbbd0d0, 0x2ac5efef, 0xe54faaaa, 0x16edfbfb, 0xc5864343, 0xd79a4d4d, 0x55663333, 0x94118585, + 0xcf8a4545, 0x10e9f9f9, 0x06040202, 0x81fe7f7f, 0xf0a05050, 0x44783c3c, 0xba259f9f, 0xe34ba8a8, + 0xf3a25151, 0xfe5da3a3, 0xc0804040, 0x8a058f8f, 0xad3f9292, 0xbc219d9d, 0x48703838, 0x04f1f5f5, + 0xdf63bcbc, 0xc177b6b6, 0x75afdada, 0x63422121, 0x30201010, 0x1ae5ffff, 0x0efdf3f3, 0x6dbfd2d2, + 0x4c81cdcd, 0x14180c0c, 0x35261313, 0x2fc3ecec, 0xe1be5f5f, 0xa2359797, 0xcc884444, 0x392e1717, + 0x5793c4c4, 0xf255a7a7, 0x82fc7e7e, 0x477a3d3d, 0xacc86464, 0xe7ba5d5d, 0x2b321919, 0x95e67373, + 0xa0c06060, 0x98198181, 0xd19e4f4f, 0x7fa3dcdc, 0x66442222, 0x7e542a2a, 0xab3b9090, 0x830b8888, + 0xca8c4646, 0x29c7eeee, 0xd36bb8b8, 0x3c281414, 0x79a7dede, 0xe2bc5e5e, 0x1d160b0b, 0x76addbdb, + 0x3bdbe0e0, 0x56643232, 0x4e743a3a, 0x1e140a0a, 0xdb924949, 0x0a0c0606, 0x6c482424, 0xe4b85c5c, + 0x5d9fc2c2, 0x6ebdd3d3, 0xef43acac, 0xa6c46262, 0xa8399191, 0xa4319595, 0x37d3e4e4, 0x8bf27979, + 0x32d5e7e7, 0x438bc8c8, 0x596e3737, 0xb7da6d6d, 0x8c018d8d, 0x64b1d5d5, 0xd29c4e4e, 0xe049a9a9, + 0xb4d86c6c, 0xfaac5656, 0x07f3f4f4, 0x25cfeaea, 0xafca6565, 0x8ef47a7a, 0xe947aeae, 0x18100808, + 0xd56fbaba, 0x88f07878, 0x6f4a2525, 0x725c2e2e, 0x24381c1c, 0xf157a6a6, 0xc773b4b4, 0x5197c6c6, + 0x23cbe8e8, 0x7ca1dddd, 0x9ce87474, 0x213e1f1f, 0xdd964b4b, 0xdc61bdbd, 0x860d8b8b, 0x850f8a8a, + 0x90e07070, 0x427c3e3e, 0xc471b5b5, 0xaacc6666, 0xd8904848, 0x05060303, 0x01f7f6f6, 0x121c0e0e, + 0xa3c26161, 0x5f6a3535, 0xf9ae5757, 0xd069b9b9, 0x91178686, 0x5899c1c1, 0x273a1d1d, 0xb9279e9e, + 0x38d9e1e1, 0x13ebf8f8, 0xb32b9898, 0x33221111, 0xbbd26969, 0x70a9d9d9, 0x89078e8e, 0xa7339494, + 0xb62d9b9b, 0x223c1e1e, 0x92158787, 0x20c9e9e9, 0x4987cece, 0xffaa5555, 0x78502828, 0x7aa5dfdf, + 0x8f038c8c, 0xf859a1a1, 0x80098989, 0x171a0d0d, 0xda65bfbf, 0x31d7e6e6, 0xc6844242, 0xb8d06868, + 0xc3824141, 0xb0299999, 0x775a2d2d, 0x111e0f0f, 0xcb7bb0b0, 0xfca85454, 0xd66dbbbb, 0x3a2c1616, + }, + { + 0x63a5c663, 0x7c84f87c, 0x7799ee77, 0x7b8df67b, 0xf20dfff2, 0x6bbdd66b, 0x6fb1de6f, 0xc55491c5, + 0x30506030, 0x01030201, 0x67a9ce67, 0x2b7d562b, 0xfe19e7fe, 0xd762b5d7, 0xabe64dab, 0x769aec76, + 0xca458fca, 0x829d1f82, 0xc94089c9, 0x7d87fa7d, 0xfa15effa, 0x59ebb259, 0x47c98e47, 0xf00bfbf0, + 0xadec41ad, 0xd467b3d4, 0xa2fd5fa2, 0xafea45af, 0x9cbf239c, 0xa4f753a4, 0x7296e472, 0xc05b9bc0, + 0xb7c275b7, 0xfd1ce1fd, 0x93ae3d93, 0x266a4c26, 0x365a6c36, 0x3f417e3f, 0xf702f5f7, 0xcc4f83cc, + 0x345c6834, 0xa5f451a5, 0xe534d1e5, 0xf108f9f1, 0x7193e271, 0xd873abd8, 0x31536231, 0x153f2a15, + 0x040c0804, 0xc75295c7, 0x23654623, 0xc35e9dc3, 0x18283018, 0x96a13796, 0x050f0a05, 0x9ab52f9a, + 0x07090e07, 0x12362412, 0x809b1b80, 0xe23ddfe2, 0xeb26cdeb, 0x27694e27, 0xb2cd7fb2, 0x759fea75, + 0x091b1209, 0x839e1d83, 0x2c74582c, 0x1a2e341a, 0x1b2d361b, 0x6eb2dc6e, 0x5aeeb45a, 0xa0fb5ba0, + 0x52f6a452, 0x3b4d763b, 0xd661b7d6, 0xb3ce7db3, 0x297b5229, 0xe33edde3, 0x2f715e2f, 0x84971384, + 0x53f5a653, 0xd168b9d1, 0x00000000, 0xed2cc1ed, 0x20604020, 0xfc1fe3fc, 0xb1c879b1, 0x5bedb65b, + 0x6abed46a, 0xcb468dcb, 0xbed967be, 0x394b7239, 0x4ade944a, 0x4cd4984c, 0x58e8b058, 0xcf4a85cf, + 0xd06bbbd0, 0xef2ac5ef, 0xaae54faa, 0xfb16edfb, 0x43c58643, 0x4dd79a4d, 0x33556633, 0x85941185, + 0x45cf8a45, 0xf910e9f9, 0x02060402, 0x7f81fe7f, 0x50f0a050, 0x3c44783c, 0x9fba259f, 0xa8e34ba8, + 0x51f3a251, 0xa3fe5da3, 0x40c08040, 0x8f8a058f, 0x92ad3f92, 0x9dbc219d, 0x38487038, 0xf504f1f5, + 0xbcdf63bc, 0xb6c177b6, 0xda75afda, 0x21634221, 0x10302010, 0xff1ae5ff, 0xf30efdf3, 0xd26dbfd2, + 0xcd4c81cd, 0x0c14180c, 0x13352613, 0xec2fc3ec, 0x5fe1be5f, 0x97a23597, 0x44cc8844, 0x17392e17, + 0xc45793c4, 0xa7f255a7, 0x7e82fc7e, 0x3d477a3d, 0x64acc864, 0x5de7ba5d, 0x192b3219, 0x7395e673, + 0x60a0c060, 0x81981981, 0x4fd19e4f, 0xdc7fa3dc, 0x22664422, 0x2a7e542a, 0x90ab3b90, 0x88830b88, + 0x46ca8c46, 0xee29c7ee, 0xb8d36bb8, 0x143c2814, 0xde79a7de, 0x5ee2bc5e, 0x0b1d160b, 0xdb76addb, + 0xe03bdbe0, 0x32566432, 0x3a4e743a, 0x0a1e140a, 0x49db9249, 0x060a0c06, 0x246c4824, 0x5ce4b85c, + 0xc25d9fc2, 0xd36ebdd3, 0xacef43ac, 0x62a6c462, 0x91a83991, 0x95a43195, 0xe437d3e4, 0x798bf279, + 0xe732d5e7, 0xc8438bc8, 0x37596e37, 0x6db7da6d, 0x8d8c018d, 0xd564b1d5, 0x4ed29c4e, 0xa9e049a9, + 0x6cb4d86c, 0x56faac56, 0xf407f3f4, 0xea25cfea, 0x65afca65, 0x7a8ef47a, 0xaee947ae, 0x08181008, + 0xbad56fba, 0x7888f078, 0x256f4a25, 0x2e725c2e, 0x1c24381c, 0xa6f157a6, 0xb4c773b4, 0xc65197c6, + 0xe823cbe8, 0xdd7ca1dd, 0x749ce874, 0x1f213e1f, 0x4bdd964b, 0xbddc61bd, 0x8b860d8b, 0x8a850f8a, + 0x7090e070, 0x3e427c3e, 0xb5c471b5, 0x66aacc66, 0x48d89048, 0x03050603, 0xf601f7f6, 0x0e121c0e, + 0x61a3c261, 0x355f6a35, 0x57f9ae57, 0xb9d069b9, 0x86911786, 0xc15899c1, 0x1d273a1d, 0x9eb9279e, + 0xe138d9e1, 0xf813ebf8, 0x98b32b98, 0x11332211, 0x69bbd269, 0xd970a9d9, 0x8e89078e, 0x94a73394, + 0x9bb62d9b, 0x1e223c1e, 0x87921587, 0xe920c9e9, 0xce4987ce, 0x55ffaa55, 0x28785028, 0xdf7aa5df, + 0x8c8f038c, 0xa1f859a1, 0x89800989, 0x0d171a0d, 0xbfda65bf, 0xe631d7e6, 0x42c68442, 0x68b8d068, + 0x41c38241, 0x99b02999, 0x2d775a2d, 0x0f111e0f, 0xb0cb7bb0, 0x54fca854, 0xbbd66dbb, 0x163a2c16, + }, + { + 0x6363a5c6, 0x7c7c84f8, 0x777799ee, 0x7b7b8df6, 0xf2f20dff, 0x6b6bbdd6, 0x6f6fb1de, 0xc5c55491, + 0x30305060, 0x01010302, 0x6767a9ce, 0x2b2b7d56, 0xfefe19e7, 0xd7d762b5, 0xababe64d, 0x76769aec, + 0xcaca458f, 0x82829d1f, 0xc9c94089, 0x7d7d87fa, 0xfafa15ef, 0x5959ebb2, 0x4747c98e, 0xf0f00bfb, + 0xadadec41, 0xd4d467b3, 0xa2a2fd5f, 0xafafea45, 0x9c9cbf23, 0xa4a4f753, 0x727296e4, 0xc0c05b9b, + 0xb7b7c275, 0xfdfd1ce1, 0x9393ae3d, 0x26266a4c, 0x36365a6c, 0x3f3f417e, 0xf7f702f5, 0xcccc4f83, + 0x34345c68, 0xa5a5f451, 0xe5e534d1, 0xf1f108f9, 0x717193e2, 0xd8d873ab, 0x31315362, 0x15153f2a, + 0x04040c08, 0xc7c75295, 0x23236546, 0xc3c35e9d, 0x18182830, 0x9696a137, 0x05050f0a, 0x9a9ab52f, + 0x0707090e, 0x12123624, 0x80809b1b, 0xe2e23ddf, 0xebeb26cd, 0x2727694e, 0xb2b2cd7f, 0x75759fea, + 0x09091b12, 0x83839e1d, 0x2c2c7458, 0x1a1a2e34, 0x1b1b2d36, 0x6e6eb2dc, 0x5a5aeeb4, 0xa0a0fb5b, + 0x5252f6a4, 0x3b3b4d76, 0xd6d661b7, 0xb3b3ce7d, 0x29297b52, 0xe3e33edd, 0x2f2f715e, 0x84849713, + 0x5353f5a6, 0xd1d168b9, 0x00000000, 0xeded2cc1, 0x20206040, 0xfcfc1fe3, 0xb1b1c879, 0x5b5bedb6, + 0x6a6abed4, 0xcbcb468d, 0xbebed967, 0x39394b72, 0x4a4ade94, 0x4c4cd498, 0x5858e8b0, 0xcfcf4a85, + 0xd0d06bbb, 0xefef2ac5, 0xaaaae54f, 0xfbfb16ed, 0x4343c586, 0x4d4dd79a, 0x33335566, 0x85859411, + 0x4545cf8a, 0xf9f910e9, 0x02020604, 0x7f7f81fe, 0x5050f0a0, 0x3c3c4478, 0x9f9fba25, 0xa8a8e34b, + 0x5151f3a2, 0xa3a3fe5d, 0x4040c080, 0x8f8f8a05, 0x9292ad3f, 0x9d9dbc21, 0x38384870, 0xf5f504f1, + 0xbcbcdf63, 0xb6b6c177, 0xdada75af, 0x21216342, 0x10103020, 0xffff1ae5, 0xf3f30efd, 0xd2d26dbf, + 0xcdcd4c81, 0x0c0c1418, 0x13133526, 0xecec2fc3, 0x5f5fe1be, 0x9797a235, 0x4444cc88, 0x1717392e, + 0xc4c45793, 0xa7a7f255, 0x7e7e82fc, 0x3d3d477a, 0x6464acc8, 0x5d5de7ba, 0x19192b32, 0x737395e6, + 0x6060a0c0, 0x81819819, 0x4f4fd19e, 0xdcdc7fa3, 0x22226644, 0x2a2a7e54, 0x9090ab3b, 0x8888830b, + 0x4646ca8c, 0xeeee29c7, 0xb8b8d36b, 0x14143c28, 0xdede79a7, 0x5e5ee2bc, 0x0b0b1d16, 0xdbdb76ad, + 0xe0e03bdb, 0x32325664, 0x3a3a4e74, 0x0a0a1e14, 0x4949db92, 0x06060a0c, 0x24246c48, 0x5c5ce4b8, + 0xc2c25d9f, 0xd3d36ebd, 0xacacef43, 0x6262a6c4, 0x9191a839, 0x9595a431, 0xe4e437d3, 0x79798bf2, + 0xe7e732d5, 0xc8c8438b, 0x3737596e, 0x6d6db7da, 0x8d8d8c01, 0xd5d564b1, 0x4e4ed29c, 0xa9a9e049, + 0x6c6cb4d8, 0x5656faac, 0xf4f407f3, 0xeaea25cf, 0x6565afca, 0x7a7a8ef4, 0xaeaee947, 0x08081810, + 0xbabad56f, 0x787888f0, 0x25256f4a, 0x2e2e725c, 0x1c1c2438, 0xa6a6f157, 0xb4b4c773, 0xc6c65197, + 0xe8e823cb, 0xdddd7ca1, 0x74749ce8, 0x1f1f213e, 0x4b4bdd96, 0xbdbddc61, 0x8b8b860d, 0x8a8a850f, + 0x707090e0, 0x3e3e427c, 0xb5b5c471, 0x6666aacc, 0x4848d890, 0x03030506, 0xf6f601f7, 0x0e0e121c, + 0x6161a3c2, 0x35355f6a, 0x5757f9ae, 0xb9b9d069, 0x86869117, 0xc1c15899, 0x1d1d273a, 0x9e9eb927, + 0xe1e138d9, 0xf8f813eb, 0x9898b32b, 0x11113322, 0x6969bbd2, 0xd9d970a9, 0x8e8e8907, 0x9494a733, + 0x9b9bb62d, 0x1e1e223c, 0x87879215, 0xe9e920c9, 0xcece4987, 0x5555ffaa, 0x28287850, 0xdfdf7aa5, + 0x8c8c8f03, 0xa1a1f859, 0x89898009, 0x0d0d171a, 0xbfbfda65, 0xe6e631d7, 0x4242c684, 0x6868b8d0, + 0x4141c382, 0x9999b029, 0x2d2d775a, 0x0f0f111e, 0xb0b0cb7b, 0x5454fca8, 0xbbbbd66d, 0x16163a2c, + }, +} + +// Lookup tables for decryption. +// These can be recomputed by adapting the tests in aes_test.go. + +var td = [4][256]uint32{ + { + 0x51f4a750, 0x7e416553, 0x1a17a4c3, 0x3a275e96, 0x3bab6bcb, 0x1f9d45f1, 0xacfa58ab, 0x4be30393, + 0x2030fa55, 0xad766df6, 0x88cc7691, 0xf5024c25, 0x4fe5d7fc, 0xc52acbd7, 0x26354480, 0xb562a38f, + 0xdeb15a49, 0x25ba1b67, 0x45ea0e98, 0x5dfec0e1, 0xc32f7502, 0x814cf012, 0x8d4697a3, 0x6bd3f9c6, + 0x038f5fe7, 0x15929c95, 0xbf6d7aeb, 0x955259da, 0xd4be832d, 0x587421d3, 0x49e06929, 0x8ec9c844, + 0x75c2896a, 0xf48e7978, 0x99583e6b, 0x27b971dd, 0xbee14fb6, 0xf088ad17, 0xc920ac66, 0x7dce3ab4, + 0x63df4a18, 0xe51a3182, 0x97513360, 0x62537f45, 0xb16477e0, 0xbb6bae84, 0xfe81a01c, 0xf9082b94, + 0x70486858, 0x8f45fd19, 0x94de6c87, 0x527bf8b7, 0xab73d323, 0x724b02e2, 0xe31f8f57, 0x6655ab2a, + 0xb2eb2807, 0x2fb5c203, 0x86c57b9a, 0xd33708a5, 0x302887f2, 0x23bfa5b2, 0x02036aba, 0xed16825c, + 0x8acf1c2b, 0xa779b492, 0xf307f2f0, 0x4e69e2a1, 0x65daf4cd, 0x0605bed5, 0xd134621f, 0xc4a6fe8a, + 0x342e539d, 0xa2f355a0, 0x058ae132, 0xa4f6eb75, 0x0b83ec39, 0x4060efaa, 0x5e719f06, 0xbd6e1051, + 0x3e218af9, 0x96dd063d, 0xdd3e05ae, 0x4de6bd46, 0x91548db5, 0x71c45d05, 0x0406d46f, 0x605015ff, + 0x1998fb24, 0xd6bde997, 0x894043cc, 0x67d99e77, 0xb0e842bd, 0x07898b88, 0xe7195b38, 0x79c8eedb, + 0xa17c0a47, 0x7c420fe9, 0xf8841ec9, 0x00000000, 0x09808683, 0x322bed48, 0x1e1170ac, 0x6c5a724e, + 0xfd0efffb, 0x0f853856, 0x3daed51e, 0x362d3927, 0x0a0fd964, 0x685ca621, 0x9b5b54d1, 0x24362e3a, + 0x0c0a67b1, 0x9357e70f, 0xb4ee96d2, 0x1b9b919e, 0x80c0c54f, 0x61dc20a2, 0x5a774b69, 0x1c121a16, + 0xe293ba0a, 0xc0a02ae5, 0x3c22e043, 0x121b171d, 0x0e090d0b, 0xf28bc7ad, 0x2db6a8b9, 0x141ea9c8, + 0x57f11985, 0xaf75074c, 0xee99ddbb, 0xa37f60fd, 0xf701269f, 0x5c72f5bc, 0x44663bc5, 0x5bfb7e34, + 0x8b432976, 0xcb23c6dc, 0xb6edfc68, 0xb8e4f163, 0xd731dcca, 0x42638510, 0x13972240, 0x84c61120, + 0x854a247d, 0xd2bb3df8, 0xaef93211, 0xc729a16d, 0x1d9e2f4b, 0xdcb230f3, 0x0d8652ec, 0x77c1e3d0, + 0x2bb3166c, 0xa970b999, 0x119448fa, 0x47e96422, 0xa8fc8cc4, 0xa0f03f1a, 0x567d2cd8, 0x223390ef, + 0x87494ec7, 0xd938d1c1, 0x8ccaa2fe, 0x98d40b36, 0xa6f581cf, 0xa57ade28, 0xdab78e26, 0x3fadbfa4, + 0x2c3a9de4, 0x5078920d, 0x6a5fcc9b, 0x547e4662, 0xf68d13c2, 0x90d8b8e8, 0x2e39f75e, 0x82c3aff5, + 0x9f5d80be, 0x69d0937c, 0x6fd52da9, 0xcf2512b3, 0xc8ac993b, 0x10187da7, 0xe89c636e, 0xdb3bbb7b, + 0xcd267809, 0x6e5918f4, 0xec9ab701, 0x834f9aa8, 0xe6956e65, 0xaaffe67e, 0x21bccf08, 0xef15e8e6, + 0xbae79bd9, 0x4a6f36ce, 0xea9f09d4, 0x29b07cd6, 0x31a4b2af, 0x2a3f2331, 0xc6a59430, 0x35a266c0, + 0x744ebc37, 0xfc82caa6, 0xe090d0b0, 0x33a7d815, 0xf104984a, 0x41ecdaf7, 0x7fcd500e, 0x1791f62f, + 0x764dd68d, 0x43efb04d, 0xccaa4d54, 0xe49604df, 0x9ed1b5e3, 0x4c6a881b, 0xc12c1fb8, 0x4665517f, + 0x9d5eea04, 0x018c355d, 0xfa877473, 0xfb0b412e, 0xb3671d5a, 0x92dbd252, 0xe9105633, 0x6dd64713, + 0x9ad7618c, 0x37a10c7a, 0x59f8148e, 0xeb133c89, 0xcea927ee, 0xb761c935, 0xe11ce5ed, 0x7a47b13c, + 0x9cd2df59, 0x55f2733f, 0x1814ce79, 0x73c737bf, 0x53f7cdea, 0x5ffdaa5b, 0xdf3d6f14, 0x7844db86, + 0xcaaff381, 0xb968c43e, 0x3824342c, 0xc2a3405f, 0x161dc372, 0xbce2250c, 0x283c498b, 0xff0d9541, + 0x39a80171, 0x080cb3de, 0xd8b4e49c, 0x6456c190, 0x7bcb8461, 0xd532b670, 0x486c5c74, 0xd0b85742, + }, + { + 0x5051f4a7, 0x537e4165, 0xc31a17a4, 0x963a275e, 0xcb3bab6b, 0xf11f9d45, 0xabacfa58, 0x934be303, + 0x552030fa, 0xf6ad766d, 0x9188cc76, 0x25f5024c, 0xfc4fe5d7, 0xd7c52acb, 0x80263544, 0x8fb562a3, + 0x49deb15a, 0x6725ba1b, 0x9845ea0e, 0xe15dfec0, 0x02c32f75, 0x12814cf0, 0xa38d4697, 0xc66bd3f9, + 0xe7038f5f, 0x9515929c, 0xebbf6d7a, 0xda955259, 0x2dd4be83, 0xd3587421, 0x2949e069, 0x448ec9c8, + 0x6a75c289, 0x78f48e79, 0x6b99583e, 0xdd27b971, 0xb6bee14f, 0x17f088ad, 0x66c920ac, 0xb47dce3a, + 0x1863df4a, 0x82e51a31, 0x60975133, 0x4562537f, 0xe0b16477, 0x84bb6bae, 0x1cfe81a0, 0x94f9082b, + 0x58704868, 0x198f45fd, 0x8794de6c, 0xb7527bf8, 0x23ab73d3, 0xe2724b02, 0x57e31f8f, 0x2a6655ab, + 0x07b2eb28, 0x032fb5c2, 0x9a86c57b, 0xa5d33708, 0xf2302887, 0xb223bfa5, 0xba02036a, 0x5ced1682, + 0x2b8acf1c, 0x92a779b4, 0xf0f307f2, 0xa14e69e2, 0xcd65daf4, 0xd50605be, 0x1fd13462, 0x8ac4a6fe, + 0x9d342e53, 0xa0a2f355, 0x32058ae1, 0x75a4f6eb, 0x390b83ec, 0xaa4060ef, 0x065e719f, 0x51bd6e10, + 0xf93e218a, 0x3d96dd06, 0xaedd3e05, 0x464de6bd, 0xb591548d, 0x0571c45d, 0x6f0406d4, 0xff605015, + 0x241998fb, 0x97d6bde9, 0xcc894043, 0x7767d99e, 0xbdb0e842, 0x8807898b, 0x38e7195b, 0xdb79c8ee, + 0x47a17c0a, 0xe97c420f, 0xc9f8841e, 0x00000000, 0x83098086, 0x48322bed, 0xac1e1170, 0x4e6c5a72, + 0xfbfd0eff, 0x560f8538, 0x1e3daed5, 0x27362d39, 0x640a0fd9, 0x21685ca6, 0xd19b5b54, 0x3a24362e, + 0xb10c0a67, 0x0f9357e7, 0xd2b4ee96, 0x9e1b9b91, 0x4f80c0c5, 0xa261dc20, 0x695a774b, 0x161c121a, + 0x0ae293ba, 0xe5c0a02a, 0x433c22e0, 0x1d121b17, 0x0b0e090d, 0xadf28bc7, 0xb92db6a8, 0xc8141ea9, + 0x8557f119, 0x4caf7507, 0xbbee99dd, 0xfda37f60, 0x9ff70126, 0xbc5c72f5, 0xc544663b, 0x345bfb7e, + 0x768b4329, 0xdccb23c6, 0x68b6edfc, 0x63b8e4f1, 0xcad731dc, 0x10426385, 0x40139722, 0x2084c611, + 0x7d854a24, 0xf8d2bb3d, 0x11aef932, 0x6dc729a1, 0x4b1d9e2f, 0xf3dcb230, 0xec0d8652, 0xd077c1e3, + 0x6c2bb316, 0x99a970b9, 0xfa119448, 0x2247e964, 0xc4a8fc8c, 0x1aa0f03f, 0xd8567d2c, 0xef223390, + 0xc787494e, 0xc1d938d1, 0xfe8ccaa2, 0x3698d40b, 0xcfa6f581, 0x28a57ade, 0x26dab78e, 0xa43fadbf, + 0xe42c3a9d, 0x0d507892, 0x9b6a5fcc, 0x62547e46, 0xc2f68d13, 0xe890d8b8, 0x5e2e39f7, 0xf582c3af, + 0xbe9f5d80, 0x7c69d093, 0xa96fd52d, 0xb3cf2512, 0x3bc8ac99, 0xa710187d, 0x6ee89c63, 0x7bdb3bbb, + 0x09cd2678, 0xf46e5918, 0x01ec9ab7, 0xa8834f9a, 0x65e6956e, 0x7eaaffe6, 0x0821bccf, 0xe6ef15e8, + 0xd9bae79b, 0xce4a6f36, 0xd4ea9f09, 0xd629b07c, 0xaf31a4b2, 0x312a3f23, 0x30c6a594, 0xc035a266, + 0x37744ebc, 0xa6fc82ca, 0xb0e090d0, 0x1533a7d8, 0x4af10498, 0xf741ecda, 0x0e7fcd50, 0x2f1791f6, + 0x8d764dd6, 0x4d43efb0, 0x54ccaa4d, 0xdfe49604, 0xe39ed1b5, 0x1b4c6a88, 0xb8c12c1f, 0x7f466551, + 0x049d5eea, 0x5d018c35, 0x73fa8774, 0x2efb0b41, 0x5ab3671d, 0x5292dbd2, 0x33e91056, 0x136dd647, + 0x8c9ad761, 0x7a37a10c, 0x8e59f814, 0x89eb133c, 0xeecea927, 0x35b761c9, 0xede11ce5, 0x3c7a47b1, + 0x599cd2df, 0x3f55f273, 0x791814ce, 0xbf73c737, 0xea53f7cd, 0x5b5ffdaa, 0x14df3d6f, 0x867844db, + 0x81caaff3, 0x3eb968c4, 0x2c382434, 0x5fc2a340, 0x72161dc3, 0x0cbce225, 0x8b283c49, 0x41ff0d95, + 0x7139a801, 0xde080cb3, 0x9cd8b4e4, 0x906456c1, 0x617bcb84, 0x70d532b6, 0x74486c5c, 0x42d0b857, + }, + { + 0xa75051f4, 0x65537e41, 0xa4c31a17, 0x5e963a27, 0x6bcb3bab, 0x45f11f9d, 0x58abacfa, 0x03934be3, + 0xfa552030, 0x6df6ad76, 0x769188cc, 0x4c25f502, 0xd7fc4fe5, 0xcbd7c52a, 0x44802635, 0xa38fb562, + 0x5a49deb1, 0x1b6725ba, 0x0e9845ea, 0xc0e15dfe, 0x7502c32f, 0xf012814c, 0x97a38d46, 0xf9c66bd3, + 0x5fe7038f, 0x9c951592, 0x7aebbf6d, 0x59da9552, 0x832dd4be, 0x21d35874, 0x692949e0, 0xc8448ec9, + 0x896a75c2, 0x7978f48e, 0x3e6b9958, 0x71dd27b9, 0x4fb6bee1, 0xad17f088, 0xac66c920, 0x3ab47dce, + 0x4a1863df, 0x3182e51a, 0x33609751, 0x7f456253, 0x77e0b164, 0xae84bb6b, 0xa01cfe81, 0x2b94f908, + 0x68587048, 0xfd198f45, 0x6c8794de, 0xf8b7527b, 0xd323ab73, 0x02e2724b, 0x8f57e31f, 0xab2a6655, + 0x2807b2eb, 0xc2032fb5, 0x7b9a86c5, 0x08a5d337, 0x87f23028, 0xa5b223bf, 0x6aba0203, 0x825ced16, + 0x1c2b8acf, 0xb492a779, 0xf2f0f307, 0xe2a14e69, 0xf4cd65da, 0xbed50605, 0x621fd134, 0xfe8ac4a6, + 0x539d342e, 0x55a0a2f3, 0xe132058a, 0xeb75a4f6, 0xec390b83, 0xefaa4060, 0x9f065e71, 0x1051bd6e, + 0x8af93e21, 0x063d96dd, 0x05aedd3e, 0xbd464de6, 0x8db59154, 0x5d0571c4, 0xd46f0406, 0x15ff6050, + 0xfb241998, 0xe997d6bd, 0x43cc8940, 0x9e7767d9, 0x42bdb0e8, 0x8b880789, 0x5b38e719, 0xeedb79c8, + 0x0a47a17c, 0x0fe97c42, 0x1ec9f884, 0x00000000, 0x86830980, 0xed48322b, 0x70ac1e11, 0x724e6c5a, + 0xfffbfd0e, 0x38560f85, 0xd51e3dae, 0x3927362d, 0xd9640a0f, 0xa621685c, 0x54d19b5b, 0x2e3a2436, + 0x67b10c0a, 0xe70f9357, 0x96d2b4ee, 0x919e1b9b, 0xc54f80c0, 0x20a261dc, 0x4b695a77, 0x1a161c12, + 0xba0ae293, 0x2ae5c0a0, 0xe0433c22, 0x171d121b, 0x0d0b0e09, 0xc7adf28b, 0xa8b92db6, 0xa9c8141e, + 0x198557f1, 0x074caf75, 0xddbbee99, 0x60fda37f, 0x269ff701, 0xf5bc5c72, 0x3bc54466, 0x7e345bfb, + 0x29768b43, 0xc6dccb23, 0xfc68b6ed, 0xf163b8e4, 0xdccad731, 0x85104263, 0x22401397, 0x112084c6, + 0x247d854a, 0x3df8d2bb, 0x3211aef9, 0xa16dc729, 0x2f4b1d9e, 0x30f3dcb2, 0x52ec0d86, 0xe3d077c1, + 0x166c2bb3, 0xb999a970, 0x48fa1194, 0x642247e9, 0x8cc4a8fc, 0x3f1aa0f0, 0x2cd8567d, 0x90ef2233, + 0x4ec78749, 0xd1c1d938, 0xa2fe8cca, 0x0b3698d4, 0x81cfa6f5, 0xde28a57a, 0x8e26dab7, 0xbfa43fad, + 0x9de42c3a, 0x920d5078, 0xcc9b6a5f, 0x4662547e, 0x13c2f68d, 0xb8e890d8, 0xf75e2e39, 0xaff582c3, + 0x80be9f5d, 0x937c69d0, 0x2da96fd5, 0x12b3cf25, 0x993bc8ac, 0x7da71018, 0x636ee89c, 0xbb7bdb3b, + 0x7809cd26, 0x18f46e59, 0xb701ec9a, 0x9aa8834f, 0x6e65e695, 0xe67eaaff, 0xcf0821bc, 0xe8e6ef15, + 0x9bd9bae7, 0x36ce4a6f, 0x09d4ea9f, 0x7cd629b0, 0xb2af31a4, 0x23312a3f, 0x9430c6a5, 0x66c035a2, + 0xbc37744e, 0xcaa6fc82, 0xd0b0e090, 0xd81533a7, 0x984af104, 0xdaf741ec, 0x500e7fcd, 0xf62f1791, + 0xd68d764d, 0xb04d43ef, 0x4d54ccaa, 0x04dfe496, 0xb5e39ed1, 0x881b4c6a, 0x1fb8c12c, 0x517f4665, + 0xea049d5e, 0x355d018c, 0x7473fa87, 0x412efb0b, 0x1d5ab367, 0xd25292db, 0x5633e910, 0x47136dd6, + 0x618c9ad7, 0x0c7a37a1, 0x148e59f8, 0x3c89eb13, 0x27eecea9, 0xc935b761, 0xe5ede11c, 0xb13c7a47, + 0xdf599cd2, 0x733f55f2, 0xce791814, 0x37bf73c7, 0xcdea53f7, 0xaa5b5ffd, 0x6f14df3d, 0xdb867844, + 0xf381caaf, 0xc43eb968, 0x342c3824, 0x405fc2a3, 0xc372161d, 0x250cbce2, 0x498b283c, 0x9541ff0d, + 0x017139a8, 0xb3de080c, 0xe49cd8b4, 0xc1906456, 0x84617bcb, 0xb670d532, 0x5c74486c, 0x5742d0b8, + }, + { + 0xf4a75051, 0x4165537e, 0x17a4c31a, 0x275e963a, 0xab6bcb3b, 0x9d45f11f, 0xfa58abac, 0xe303934b, + 0x30fa5520, 0x766df6ad, 0xcc769188, 0x024c25f5, 0xe5d7fc4f, 0x2acbd7c5, 0x35448026, 0x62a38fb5, + 0xb15a49de, 0xba1b6725, 0xea0e9845, 0xfec0e15d, 0x2f7502c3, 0x4cf01281, 0x4697a38d, 0xd3f9c66b, + 0x8f5fe703, 0x929c9515, 0x6d7aebbf, 0x5259da95, 0xbe832dd4, 0x7421d358, 0xe0692949, 0xc9c8448e, + 0xc2896a75, 0x8e7978f4, 0x583e6b99, 0xb971dd27, 0xe14fb6be, 0x88ad17f0, 0x20ac66c9, 0xce3ab47d, + 0xdf4a1863, 0x1a3182e5, 0x51336097, 0x537f4562, 0x6477e0b1, 0x6bae84bb, 0x81a01cfe, 0x082b94f9, + 0x48685870, 0x45fd198f, 0xde6c8794, 0x7bf8b752, 0x73d323ab, 0x4b02e272, 0x1f8f57e3, 0x55ab2a66, + 0xeb2807b2, 0xb5c2032f, 0xc57b9a86, 0x3708a5d3, 0x2887f230, 0xbfa5b223, 0x036aba02, 0x16825ced, + 0xcf1c2b8a, 0x79b492a7, 0x07f2f0f3, 0x69e2a14e, 0xdaf4cd65, 0x05bed506, 0x34621fd1, 0xa6fe8ac4, + 0x2e539d34, 0xf355a0a2, 0x8ae13205, 0xf6eb75a4, 0x83ec390b, 0x60efaa40, 0x719f065e, 0x6e1051bd, + 0x218af93e, 0xdd063d96, 0x3e05aedd, 0xe6bd464d, 0x548db591, 0xc45d0571, 0x06d46f04, 0x5015ff60, + 0x98fb2419, 0xbde997d6, 0x4043cc89, 0xd99e7767, 0xe842bdb0, 0x898b8807, 0x195b38e7, 0xc8eedb79, + 0x7c0a47a1, 0x420fe97c, 0x841ec9f8, 0x00000000, 0x80868309, 0x2bed4832, 0x1170ac1e, 0x5a724e6c, + 0x0efffbfd, 0x8538560f, 0xaed51e3d, 0x2d392736, 0x0fd9640a, 0x5ca62168, 0x5b54d19b, 0x362e3a24, + 0x0a67b10c, 0x57e70f93, 0xee96d2b4, 0x9b919e1b, 0xc0c54f80, 0xdc20a261, 0x774b695a, 0x121a161c, + 0x93ba0ae2, 0xa02ae5c0, 0x22e0433c, 0x1b171d12, 0x090d0b0e, 0x8bc7adf2, 0xb6a8b92d, 0x1ea9c814, + 0xf1198557, 0x75074caf, 0x99ddbbee, 0x7f60fda3, 0x01269ff7, 0x72f5bc5c, 0x663bc544, 0xfb7e345b, + 0x4329768b, 0x23c6dccb, 0xedfc68b6, 0xe4f163b8, 0x31dccad7, 0x63851042, 0x97224013, 0xc6112084, + 0x4a247d85, 0xbb3df8d2, 0xf93211ae, 0x29a16dc7, 0x9e2f4b1d, 0xb230f3dc, 0x8652ec0d, 0xc1e3d077, + 0xb3166c2b, 0x70b999a9, 0x9448fa11, 0xe9642247, 0xfc8cc4a8, 0xf03f1aa0, 0x7d2cd856, 0x3390ef22, + 0x494ec787, 0x38d1c1d9, 0xcaa2fe8c, 0xd40b3698, 0xf581cfa6, 0x7ade28a5, 0xb78e26da, 0xadbfa43f, + 0x3a9de42c, 0x78920d50, 0x5fcc9b6a, 0x7e466254, 0x8d13c2f6, 0xd8b8e890, 0x39f75e2e, 0xc3aff582, + 0x5d80be9f, 0xd0937c69, 0xd52da96f, 0x2512b3cf, 0xac993bc8, 0x187da710, 0x9c636ee8, 0x3bbb7bdb, + 0x267809cd, 0x5918f46e, 0x9ab701ec, 0x4f9aa883, 0x956e65e6, 0xffe67eaa, 0xbccf0821, 0x15e8e6ef, + 0xe79bd9ba, 0x6f36ce4a, 0x9f09d4ea, 0xb07cd629, 0xa4b2af31, 0x3f23312a, 0xa59430c6, 0xa266c035, + 0x4ebc3774, 0x82caa6fc, 0x90d0b0e0, 0xa7d81533, 0x04984af1, 0xecdaf741, 0xcd500e7f, 0x91f62f17, + 0x4dd68d76, 0xefb04d43, 0xaa4d54cc, 0x9604dfe4, 0xd1b5e39e, 0x6a881b4c, 0x2c1fb8c1, 0x65517f46, + 0x5eea049d, 0x8c355d01, 0x877473fa, 0x0b412efb, 0x671d5ab3, 0xdbd25292, 0x105633e9, 0xd647136d, + 0xd7618c9a, 0xa10c7a37, 0xf8148e59, 0x133c89eb, 0xa927eece, 0x61c935b7, 0x1ce5ede1, 0x47b13c7a, + 0xd2df599c, 0xf2733f55, 0x14ce7918, 0xc737bf73, 0xf7cdea53, 0xfdaa5b5f, 0x3d6f14df, 0x44db8678, + 0xaff381ca, 0x68c43eb9, 0x24342c38, 0xa3405fc2, 0x1dc37216, 0xe2250cbc, 0x3c498b28, 0x0d9541ff, + 0xa8017139, 0x0cb3de08, 0xb4e49cd8, 0x56c19064, 0xcb84617b, 0x32b670d5, 0x6c5c7448, 0xb85742d0, + }, +} diff --git a/libgo/go/crypto/block/cbc.go b/libgo/go/crypto/block/cbc.go new file mode 100644 index 000000000..23229c09f --- /dev/null +++ b/libgo/go/crypto/block/cbc.go @@ -0,0 +1,71 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Cipher block chaining (CBC) mode. + +// CBC provides confidentiality by xoring (chaining) each plaintext block +// with the previous ciphertext block before applying the block cipher. + +// See NIST SP 800-38A, pp 10-11 + +package block + +import ( + "io" +) + +type cbcCipher struct { + c Cipher + blockSize int + iv []byte + tmp []byte +} + +func newCBC(c Cipher, iv []byte) *cbcCipher { + n := c.BlockSize() + x := new(cbcCipher) + x.c = c + x.blockSize = n + x.iv = dup(iv) + x.tmp = make([]byte, n) + return x +} + +func (x *cbcCipher) BlockSize() int { return x.blockSize } + +func (x *cbcCipher) Encrypt(dst, src []byte) { + for i := 0; i < x.blockSize; i++ { + x.iv[i] ^= src[i] + } + x.c.Encrypt(x.iv, x.iv) + for i := 0; i < x.blockSize; i++ { + dst[i] = x.iv[i] + } +} + +func (x *cbcCipher) Decrypt(dst, src []byte) { + x.c.Decrypt(x.tmp, src) + for i := 0; i < x.blockSize; i++ { + x.tmp[i] ^= x.iv[i] + x.iv[i] = src[i] + dst[i] = x.tmp[i] + } +} + +// NewCBCDecrypter returns a reader that reads data from r and decrypts it using c +// in cipher block chaining (CBC) mode with the initialization vector iv. +// The returned Reader does not buffer or read ahead except +// as required by the cipher's block size. +func NewCBCDecrypter(c Cipher, iv []byte, r io.Reader) io.Reader { + return NewECBDecrypter(newCBC(c, iv), r) +} + +// NewCBCEncrypter returns a writer that encrypts data using c +// in cipher block chaining (CBC) mode with the initialization vector iv +// and writes the encrypted data to w. +// The returned Writer does no buffering except as required +// by the cipher's block size, so there is no need for a Flush method. +func NewCBCEncrypter(c Cipher, iv []byte, w io.Writer) io.Writer { + return NewECBEncrypter(newCBC(c, iv), w) +} diff --git a/libgo/go/crypto/block/cfb.go b/libgo/go/crypto/block/cfb.go new file mode 100644 index 000000000..f20c0a04f --- /dev/null +++ b/libgo/go/crypto/block/cfb.go @@ -0,0 +1,96 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Cipher feedback (CFB) mode. + +// CFB provides confidentiality by feeding a fraction of +// the previous ciphertext in as the plaintext for the next +// block operation. + +// See NIST SP 800-38A, pp 11-13 + +package block + +import ( + "io" +) + +type cfbCipher struct { + c Cipher + blockSize int // our block size (s/8) + cipherSize int // underlying cipher block size + iv []byte + tmp []byte +} + +func newCFB(c Cipher, s int, iv []byte) *cfbCipher { + if s == 0 || s%8 != 0 { + panic("crypto/block: invalid CFB mode") + } + b := c.BlockSize() + x := new(cfbCipher) + x.c = c + x.blockSize = s / 8 + x.cipherSize = b + x.iv = dup(iv) + x.tmp = make([]byte, b) + return x +} + +func (x *cfbCipher) BlockSize() int { return x.blockSize } + +func (x *cfbCipher) Encrypt(dst, src []byte) { + // Encrypt old IV and xor prefix with src to make dst. + x.c.Encrypt(x.tmp, x.iv) + for i := 0; i < x.blockSize; i++ { + dst[i] = src[i] ^ x.tmp[i] + } + + // Slide unused IV pieces down and insert dst at end. + for i := 0; i < x.cipherSize-x.blockSize; i++ { + x.iv[i] = x.iv[i+x.blockSize] + } + off := x.cipherSize - x.blockSize + for i := off; i < x.cipherSize; i++ { + x.iv[i] = dst[i-off] + } +} + +func (x *cfbCipher) Decrypt(dst, src []byte) { + // Encrypt [sic] old IV and xor prefix with src to make dst. + x.c.Encrypt(x.tmp, x.iv) + for i := 0; i < x.blockSize; i++ { + dst[i] = src[i] ^ x.tmp[i] + } + + // Slide unused IV pieces down and insert src at top. + for i := 0; i < x.cipherSize-x.blockSize; i++ { + x.iv[i] = x.iv[i+x.blockSize] + } + off := x.cipherSize - x.blockSize + for i := off; i < x.cipherSize; i++ { + // Reconstruct src = dst ^ x.tmp + // in case we overwrote src (src == dst). + x.iv[i] = dst[i-off] ^ x.tmp[i-off] + } +} + +// NewCFBDecrypter returns a reader that reads data from r and decrypts it using c +// in s-bit cipher feedback (CFB) mode with the initialization vector iv. +// The returned Reader does not buffer or read ahead except +// as required by the cipher's block size. +// Modes for s not a multiple of 8 are unimplemented. +func NewCFBDecrypter(c Cipher, s int, iv []byte, r io.Reader) io.Reader { + return NewECBDecrypter(newCFB(c, s, iv), r) +} + +// NewCFBEncrypter returns a writer that encrypts data using c +// in s-bit cipher feedback (CFB) mode with the initialization vector iv +// and writes the encrypted data to w. +// The returned Writer does no buffering except as required +// by the cipher's block size, so there is no need for a Flush method. +// Modes for s not a multiple of 8 are unimplemented. +func NewCFBEncrypter(c Cipher, s int, iv []byte, w io.Writer) io.Writer { + return NewECBEncrypter(newCFB(c, s, iv), w) +} diff --git a/libgo/go/crypto/block/cfb_aes_test.go b/libgo/go/crypto/block/cfb_aes_test.go new file mode 100644 index 000000000..e400c182a --- /dev/null +++ b/libgo/go/crypto/block/cfb_aes_test.go @@ -0,0 +1,311 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// CFB AES test vectors. + +// See U.S. National Institute of Standards and Technology (NIST) +// Special Publication 800-38A, ``Recommendation for Block Cipher +// Modes of Operation,'' 2001 Edition, pp. 29-52. + +package block + +import ( + "bytes" + "crypto/aes" + "io" + "testing" +) + +type cfbTest struct { + name string + s int + key []byte + iv []byte + in []byte + out []byte +} + +var cfbAESTests = []cfbTest{ + { + "CFB1-AES128", + 1, + commonKey128, + commonIV, + []byte{ + 0<<7 | 1<<6 | 1<<5 | 0<<4 | 1<<3 | 0<<2 | 1<<1, + 1<<7 | 1<<6 | 0<<5 | 0<<4 | 0<<3 | 0<<2 | 0<<1, + }, + []byte{ + 0<<7 | 1<<6 | 1<<5 | 0<<4 | 1<<3 | 0<<2 | 0<<1, + 1<<7 | 0<<6 | 1<<5 | 1<<4 | 0<<3 | 0<<2 | 1<<1, + }, + }, + { + "CFB1-AES192", + 1, + commonKey192, + commonIV, + []byte{ + 0<<7 | 1<<6 | 1<<5 | 0<<4 | 1<<3 | 0<<2 | 1<<1, + 1<<7 | 1<<6 | 0<<5 | 0<<4 | 0<<3 | 0<<2 | 0<<1, + }, + []byte{ + 1<<7 | 0<<6 | 0<<5 | 1<<4 | 0<<3 | 0<<2 | 1<<1, + 0<<7 | 1<<6 | 0<<5 | 1<<4 | 1<<3 | 0<<2 | 0<<1, + }, + }, + { + "CFB1-AES256", + 1, + commonKey256, + commonIV, + []byte{ + 0<<7 | 1<<6 | 1<<5 | 0<<4 | 1<<3 | 0<<2 | 1<<1, + 1<<7 | 1<<6 | 0<<5 | 0<<4 | 0<<3 | 0<<2 | 0<<1, + }, + []byte{ + 1<<7 | 0<<6 | 0<<5 | 1<<4 | 0<<3 | 0<<2 | 0<<1, + 0<<7 | 0<<6 | 1<<5 | 0<<4 | 1<<3 | 0<<2 | 0<<1, + }, + }, + + { + "CFB8-AES128", + 8, + commonKey128, + commonIV, + []byte{ + 0x6b, + 0xc1, + 0xbe, + 0xe2, + 0x2e, + 0x40, + 0x9f, + 0x96, + 0xe9, + 0x3d, + 0x7e, + 0x11, + 0x73, + 0x93, + 0x17, + 0x2a, + 0xae, + 0x2d, + }, + []byte{ + 0x3b, + 0x79, + 0x42, + 0x4c, + 0x9c, + 0x0d, + 0xd4, + 0x36, + 0xba, + 0xce, + 0x9e, + 0x0e, + 0xd4, + 0x58, + 0x6a, + 0x4f, + 0x32, + 0xb9, + }, + }, + + { + "CFB8-AES192", + 8, + commonKey192, + commonIV, + []byte{ + 0x6b, + 0xc1, + 0xbe, + 0xe2, + 0x2e, + 0x40, + 0x9f, + 0x96, + 0xe9, + 0x3d, + 0x7e, + 0x11, + 0x73, + 0x93, + 0x17, + 0x2a, + 0xae, + 0x2d, + }, + []byte{ + 0xcd, + 0xa2, + 0x52, + 0x1e, + 0xf0, + 0xa9, + 0x05, + 0xca, + 0x44, + 0xcd, + 0x05, + 0x7c, + 0xbf, + 0x0d, + 0x47, + 0xa0, + 0x67, + 0x8a, + }, + }, + + { + "CFB8-AES256", + 8, + commonKey256, + commonIV, + []byte{ + 0x6b, + 0xc1, + 0xbe, + 0xe2, + 0x2e, + 0x40, + 0x9f, + 0x96, + 0xe9, + 0x3d, + 0x7e, + 0x11, + 0x73, + 0x93, + 0x17, + 0x2a, + 0xae, + 0x2d, + }, + []byte{ + 0xdc, + 0x1f, + 0x1a, + 0x85, + 0x20, + 0xa6, + 0x4d, + 0xb5, + 0x5f, + 0xcc, + 0x8a, + 0xc5, + 0x54, + 0x84, + 0x4e, + 0x88, + 0x97, + 0x00, + }, + }, + + { + "CFB128-AES128", + 128, + commonKey128, + commonIV, + []byte{ + 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a, + 0xae, 0x2d, 0x8a, 0x57, 0x1e, 0x03, 0xac, 0x9c, 0x9e, 0xb7, 0x6f, 0xac, 0x45, 0xaf, 0x8e, 0x51, + 0x30, 0xc8, 0x1c, 0x46, 0xa3, 0x5c, 0xe4, 0x11, 0xe5, 0xfb, 0xc1, 0x19, 0x1a, 0x0a, 0x52, 0xef, + 0xf6, 0x9f, 0x24, 0x45, 0xdf, 0x4f, 0x9b, 0x17, 0xad, 0x2b, 0x41, 0x7b, 0xe6, 0x6c, 0x37, 0x10, + }, + []byte{ + 0x3b, 0x3f, 0xd9, 0x2e, 0xb7, 0x2d, 0xad, 0x20, 0x33, 0x34, 0x49, 0xf8, 0xe8, 0x3c, 0xfb, 0x4a, + 0xc8, 0xa6, 0x45, 0x37, 0xa0, 0xb3, 0xa9, 0x3f, 0xcd, 0xe3, 0xcd, 0xad, 0x9f, 0x1c, 0xe5, 0x8b, + 0x26, 0x75, 0x1f, 0x67, 0xa3, 0xcb, 0xb1, 0x40, 0xb1, 0x80, 0x8c, 0xf1, 0x87, 0xa4, 0xf4, 0xdf, + 0xc0, 0x4b, 0x05, 0x35, 0x7c, 0x5d, 0x1c, 0x0e, 0xea, 0xc4, 0xc6, 0x6f, 0x9f, 0xf7, 0xf2, 0xe6, + }, + }, + + { + "CFB128-AES192", + 128, + commonKey192, + commonIV, + []byte{ + 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a, + 0xae, 0x2d, 0x8a, 0x57, 0x1e, 0x03, 0xac, 0x9c, 0x9e, 0xb7, 0x6f, 0xac, 0x45, 0xaf, 0x8e, 0x51, + 0x30, 0xc8, 0x1c, 0x46, 0xa3, 0x5c, 0xe4, 0x11, 0xe5, 0xfb, 0xc1, 0x19, 0x1a, 0x0a, 0x52, 0xef, + 0xf6, 0x9f, 0x24, 0x45, 0xdf, 0x4f, 0x9b, 0x17, 0xad, 0x2b, 0x41, 0x7b, 0xe6, 0x6c, 0x37, 0x10, + }, + []byte{ + 0xcd, 0xc8, 0x0d, 0x6f, 0xdd, 0xf1, 0x8c, 0xab, 0x34, 0xc2, 0x59, 0x09, 0xc9, 0x9a, 0x41, 0x74, + 0x67, 0xce, 0x7f, 0x7f, 0x81, 0x17, 0x36, 0x21, 0x96, 0x1a, 0x2b, 0x70, 0x17, 0x1d, 0x3d, 0x7a, + 0x2e, 0x1e, 0x8a, 0x1d, 0xd5, 0x9b, 0x88, 0xb1, 0xc8, 0xe6, 0x0f, 0xed, 0x1e, 0xfa, 0xc4, 0xc9, + 0xc0, 0x5f, 0x9f, 0x9c, 0xa9, 0x83, 0x4f, 0xa0, 0x42, 0xae, 0x8f, 0xba, 0x58, 0x4b, 0x09, 0xff, + }, + }, + + { + "CFB128-AES256", + 128, + commonKey256, + commonIV, + []byte{ + 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a, + 0xae, 0x2d, 0x8a, 0x57, 0x1e, 0x03, 0xac, 0x9c, 0x9e, 0xb7, 0x6f, 0xac, 0x45, 0xaf, 0x8e, 0x51, + 0x30, 0xc8, 0x1c, 0x46, 0xa3, 0x5c, 0xe4, 0x11, 0xe5, 0xfb, 0xc1, 0x19, 0x1a, 0x0a, 0x52, 0xef, + 0xf6, 0x9f, 0x24, 0x45, 0xdf, 0x4f, 0x9b, 0x17, 0xad, 0x2b, 0x41, 0x7b, 0xe6, 0x6c, 0x37, 0x10, + }, + []byte{ + 0xdc, 0x7e, 0x84, 0xbf, 0xda, 0x79, 0x16, 0x4b, 0x7e, 0xcd, 0x84, 0x86, 0x98, 0x5d, 0x38, 0x60, + 0x39, 0xff, 0xed, 0x14, 0x3b, 0x28, 0xb1, 0xc8, 0x32, 0x11, 0x3c, 0x63, 0x31, 0xe5, 0x40, 0x7b, + 0xdf, 0x10, 0x13, 0x24, 0x15, 0xe5, 0x4b, 0x92, 0xa1, 0x3e, 0xd0, 0xa8, 0x26, 0x7a, 0xe2, 0xf9, + 0x75, 0xa3, 0x85, 0x74, 0x1a, 0xb9, 0xce, 0xf8, 0x20, 0x31, 0x62, 0x3d, 0x55, 0xb1, 0xe4, 0x71, + }, + }, +} + +func TestCFB_AES(t *testing.T) { + for _, tt := range cfbAESTests { + test := tt.name + + if tt.s == 1 { + // 1-bit CFB not implemented + continue + } + + c, err := aes.NewCipher(tt.key) + if err != nil { + t.Errorf("%s: NewCipher(%d bytes) = %s", test, len(tt.key), err) + continue + } + + var crypt bytes.Buffer + w := NewCFBEncrypter(c, tt.s, tt.iv, &crypt) + var r io.Reader = bytes.NewBuffer(tt.in) + n, err := io.Copy(w, r) + if n != int64(len(tt.in)) || err != nil { + t.Errorf("%s: CFBEncrypter io.Copy = %d, %v want %d, nil", test, n, err, len(tt.in)) + } else if d := crypt.Bytes(); !same(tt.out, d) { + t.Errorf("%s: CFBEncrypter\nhave %x\nwant %x", test, d, tt.out) + } + + var plain bytes.Buffer + r = NewCFBDecrypter(c, tt.s, tt.iv, bytes.NewBuffer(tt.out)) + w = &plain + n, err = io.Copy(w, r) + if n != int64(len(tt.out)) || err != nil { + t.Errorf("%s: CFBDecrypter io.Copy = %d, %v want %d, nil", test, n, err, len(tt.out)) + } else if d := plain.Bytes(); !same(tt.in, d) { + t.Errorf("%s: CFBDecrypter\nhave %x\nwant %x", test, d, tt.in) + } + + if t.Failed() { + break + } + } +} diff --git a/libgo/go/crypto/block/cipher.go b/libgo/go/crypto/block/cipher.go new file mode 100644 index 000000000..e1099e9a1 --- /dev/null +++ b/libgo/go/crypto/block/cipher.go @@ -0,0 +1,57 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The block package is deprecated, use cipher instead. +// The block package implements standard block cipher modes +// that can be wrapped around low-level block cipher implementations. +// See http://csrc.nist.gov/groups/ST/toolkit/BCM/current_modes.html +// and NIST Special Publication 800-38A. +package block + +// A Cipher represents an implementation of block cipher +// using a given key. It provides the capability to encrypt +// or decrypt individual blocks. The mode implementations +// extend that capability to streams of blocks. +type Cipher interface { + // BlockSize returns the cipher's block size. + BlockSize() int + + // Encrypt encrypts the first block in src into dst. + // Src and dst may point at the same memory. + Encrypt(dst, src []byte) + + // Decrypt decrypts the first block in src into dst. + // Src and dst may point at the same memory. + Decrypt(dst, src []byte) +} + +// Utility routines + +func shift1(dst, src []byte) byte { + var b byte + for i := len(src) - 1; i >= 0; i-- { + bb := src[i] >> 7 + dst[i] = src[i]<<1 | b + b = bb + } + return b +} + +func same(p, q []byte) bool { + if len(p) != len(q) { + return false + } + for i := 0; i < len(p); i++ { + if p[i] != q[i] { + return false + } + } + return true +} + +func dup(p []byte) []byte { + q := make([]byte, len(p)) + copy(q, p) + return q +} diff --git a/libgo/go/crypto/block/cmac.go b/libgo/go/crypto/block/cmac.go new file mode 100644 index 000000000..b85cde72e --- /dev/null +++ b/libgo/go/crypto/block/cmac.go @@ -0,0 +1,105 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// CMAC message authentication code, defined in +// NIST Special Publication SP 800-38B. + +package block + +import ( + "hash" + "os" +) + +const ( + // minimal irreducible polynomial of degree b + r64 = 0x1b + r128 = 0x87 +) + +type cmac struct { + k1, k2, ci, digest []byte + p int // position in ci + c Cipher +} + +// TODO(rsc): Should this return an error instead of panic? + +// NewCMAC returns a new instance of a CMAC message authentication code +// digest using the given Cipher. +func NewCMAC(c Cipher) hash.Hash { + var r byte + n := c.BlockSize() + switch n { + case 64 / 8: + r = r64 + case 128 / 8: + r = r128 + default: + panic("crypto/block: NewCMAC: invalid cipher block size") + } + + d := new(cmac) + d.c = c + d.k1 = make([]byte, n) + d.k2 = make([]byte, n) + d.ci = make([]byte, n) + d.digest = make([]byte, n) + + // Subkey generation, p. 7 + c.Encrypt(d.k1, d.k1) + if shift1(d.k1, d.k1) != 0 { + d.k1[n-1] ^= r + } + if shift1(d.k2, d.k1) != 0 { + d.k2[n-1] ^= r + } + + return d +} + +// Reset clears the digest state, starting a new digest. +func (d *cmac) Reset() { + for i := range d.ci { + d.ci[i] = 0 + } + d.p = 0 +} + +// Write adds the given data to the digest state. +func (d *cmac) Write(p []byte) (n int, err os.Error) { + // Xor input into ci. + for _, c := range p { + // If ci is full, encrypt and start over. + if d.p >= len(d.ci) { + d.c.Encrypt(d.ci, d.ci) + d.p = 0 + } + d.ci[d.p] ^= c + d.p++ + } + return len(p), nil +} + +// Sum returns the CMAC digest, one cipher block in length, +// of the data written with Write. +func (d *cmac) Sum() []byte { + // Finish last block, mix in key, encrypt. + // Don't edit ci, in case caller wants + // to keep digesting after call to Sum. + k := d.k1 + if d.p < len(d.digest) { + k = d.k2 + } + for i := 0; i < len(d.ci); i++ { + d.digest[i] = d.ci[i] ^ k[i] + } + if d.p < len(d.digest) { + d.digest[d.p] ^= 0x80 + } + d.c.Encrypt(d.digest, d.digest) + return d.digest +} + +func (d *cmac) Size() int { return len(d.digest) } diff --git a/libgo/go/crypto/block/cmac_aes_test.go b/libgo/go/crypto/block/cmac_aes_test.go new file mode 100644 index 000000000..0a4a1a418 --- /dev/null +++ b/libgo/go/crypto/block/cmac_aes_test.go @@ -0,0 +1,130 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// CMAC test vectors. See NIST SP 800-38B, Appendix D. + +package block + +import ( + "crypto/aes" + "testing" +) + +type cmacAESTest struct { + key []byte + in []byte + digest []byte +} + +var cmacAESTests = []cmacAESTest{ + { + commonKey128, + nil, + []byte{0xbb, 0x1d, 0x69, 0x29, 0xe9, 0x59, 0x37, 0x28, 0x7f, 0xa3, 0x7d, 0x12, 0x9b, 0x75, 0x67, 0x46}, + }, + { + commonKey128, + []byte{0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a}, + []byte{0x07, 0x0a, 0x16, 0xb4, 0x6b, 0x4d, 0x41, 0x44, 0xf7, 0x9b, 0xdd, 0x9d, 0xd0, 0x4a, 0x28, 0x7c}, + }, + { + commonKey128, + []byte{ + 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a, + 0xae, 0x2d, 0x8a, 0x57, 0x1e, 0x03, 0xac, 0x9c, 0x9e, 0xb7, 0x6f, 0xac, 0x45, 0xaf, 0x8e, 0x51, + 0x30, 0xc8, 0x1c, 0x46, 0xa3, 0x5c, 0xe4, 0x11, + }, + []byte{0xdf, 0xa6, 0x67, 0x47, 0xde, 0x9a, 0xe6, 0x30, 0x30, 0xca, 0x32, 0x61, 0x14, 0x97, 0xc8, 0x27}, + }, + { + commonKey128, + []byte{ + 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a, + 0xae, 0x2d, 0x8a, 0x57, 0x1e, 0x03, 0xac, 0x9c, 0x9e, 0xb7, 0x6f, 0xac, 0x45, 0xaf, 0x8e, 0x51, + 0x30, 0xc8, 0x1c, 0x46, 0xa3, 0x5c, 0xe4, 0x11, 0xe5, 0xfb, 0xc1, 0x19, 0x1a, 0x0a, 0x52, 0xef, + 0xf6, 0x9f, 0x24, 0x45, 0xdf, 0x4f, 0x9b, 0x17, 0xad, 0x2b, 0x41, 0x7b, 0xe6, 0x6c, 0x37, 0x10, + }, + []byte{0x51, 0xf0, 0xbe, 0xbf, 0x7e, 0x3b, 0x9d, 0x92, 0xfc, 0x49, 0x74, 0x17, 0x79, 0x36, 0x3c, 0xfe}, + }, + { + commonKey192, + nil, + []byte{0xd1, 0x7d, 0xdf, 0x46, 0xad, 0xaa, 0xcd, 0xe5, 0x31, 0xca, 0xc4, 0x83, 0xde, 0x7a, 0x93, 0x67}, + }, + { + commonKey192, + []byte{0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a}, + []byte{0x9e, 0x99, 0xa7, 0xbf, 0x31, 0xe7, 0x10, 0x90, 0x06, 0x62, 0xf6, 0x5e, 0x61, 0x7c, 0x51, 0x84}, + }, + { + commonKey192, + []byte{ + 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a, + 0xae, 0x2d, 0x8a, 0x57, 0x1e, 0x03, 0xac, 0x9c, 0x9e, 0xb7, 0x6f, 0xac, 0x45, 0xaf, 0x8e, 0x51, + 0x30, 0xc8, 0x1c, 0x46, 0xa3, 0x5c, 0xe4, 0x11, + }, + []byte{0x8a, 0x1d, 0xe5, 0xbe, 0x2e, 0xb3, 0x1a, 0xad, 0x08, 0x9a, 0x82, 0xe6, 0xee, 0x90, 0x8b, 0x0e}, + }, + { + commonKey192, + []byte{ + 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a, + 0xae, 0x2d, 0x8a, 0x57, 0x1e, 0x03, 0xac, 0x9c, 0x9e, 0xb7, 0x6f, 0xac, 0x45, 0xaf, 0x8e, 0x51, + 0x30, 0xc8, 0x1c, 0x46, 0xa3, 0x5c, 0xe4, 0x11, 0xe5, 0xfb, 0xc1, 0x19, 0x1a, 0x0a, 0x52, 0xef, + 0xf6, 0x9f, 0x24, 0x45, 0xdf, 0x4f, 0x9b, 0x17, 0xad, 0x2b, 0x41, 0x7b, 0xe6, 0x6c, 0x37, 0x10, + }, + []byte{0xa1, 0xd5, 0xdf, 0x0e, 0xed, 0x79, 0x0f, 0x79, 0x4d, 0x77, 0x58, 0x96, 0x59, 0xf3, 0x9a, 0x11}, + }, + { + commonKey256, + nil, + []byte{0x02, 0x89, 0x62, 0xf6, 0x1b, 0x7b, 0xf8, 0x9e, 0xfc, 0x6b, 0x55, 0x1f, 0x46, 0x67, 0xd9, 0x83}, + }, + { + commonKey256, + []byte{0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a}, + []byte{0x28, 0xa7, 0x02, 0x3f, 0x45, 0x2e, 0x8f, 0x82, 0xbd, 0x4b, 0xf2, 0x8d, 0x8c, 0x37, 0xc3, 0x5c}, + }, + { + commonKey256, + []byte{ + 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a, + 0xae, 0x2d, 0x8a, 0x57, 0x1e, 0x03, 0xac, 0x9c, 0x9e, 0xb7, 0x6f, 0xac, 0x45, 0xaf, 0x8e, 0x51, + 0x30, 0xc8, 0x1c, 0x46, 0xa3, 0x5c, 0xe4, 0x11, + }, + []byte{0xaa, 0xf3, 0xd8, 0xf1, 0xde, 0x56, 0x40, 0xc2, 0x32, 0xf5, 0xb1, 0x69, 0xb9, 0xc9, 0x11, 0xe6}, + }, + { + commonKey256, + []byte{ + 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a, + 0xae, 0x2d, 0x8a, 0x57, 0x1e, 0x03, 0xac, 0x9c, 0x9e, 0xb7, 0x6f, 0xac, 0x45, 0xaf, 0x8e, 0x51, + 0x30, 0xc8, 0x1c, 0x46, 0xa3, 0x5c, 0xe4, 0x11, 0xe5, 0xfb, 0xc1, 0x19, 0x1a, 0x0a, 0x52, 0xef, + 0xf6, 0x9f, 0x24, 0x45, 0xdf, 0x4f, 0x9b, 0x17, 0xad, 0x2b, 0x41, 0x7b, 0xe6, 0x6c, 0x37, 0x10, + }, + []byte{0xe1, 0x99, 0x21, 0x90, 0x54, 0x9f, 0x6e, 0xd5, 0x69, 0x6a, 0x2c, 0x05, 0x6c, 0x31, 0x54, 0x10}, + }, +} + +func TestCMAC_AES(t *testing.T) { + for i, tt := range cmacAESTests { + c, err := aes.NewCipher(tt.key) + if err != nil { + t.Errorf("test %d: NewCipher: %s", i, err) + continue + } + d := NewCMAC(c) + n, err := d.Write(tt.in) + if err != nil || n != len(tt.in) { + t.Errorf("test %d: Write %d: %d, %s", i, len(tt.in), n, err) + continue + } + sum := d.Sum() + if !same(sum, tt.digest) { + x := d.(*cmac) + t.Errorf("test %d: digest mismatch\n\twant %x\n\thave %x\n\tk1 %x\n\tk2 %x", i, tt.digest, sum, x.k1, x.k2) + continue + } + } +} diff --git a/libgo/go/crypto/block/ctr.go b/libgo/go/crypto/block/ctr.go new file mode 100644 index 000000000..5d65c0c9a --- /dev/null +++ b/libgo/go/crypto/block/ctr.go @@ -0,0 +1,67 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Counter (CTR) mode. + +// CTR converts a block cipher into a stream cipher by +// repeatedly encrypting an incrementing counter and +// xoring the resulting stream of data with the input. + +// See NIST SP 800-38A, pp 13-15 + +package block + +import ( + "io" +) + +type ctrStream struct { + c Cipher + ctr []byte + out []byte +} + +func newCTRStream(c Cipher, ctr []byte) *ctrStream { + x := new(ctrStream) + x.c = c + x.ctr = dup(ctr) + x.out = make([]byte, len(ctr)) + return x +} + +func (x *ctrStream) Next() []byte { + // Next block is encryption of counter. + x.c.Encrypt(x.out, x.ctr) + + // Increment counter + for i := len(x.ctr) - 1; i >= 0; i-- { + x.ctr[i]++ + if x.ctr[i] != 0 { + break + } + } + + return x.out +} + +// NewCTRReader returns a reader that reads data from r, decrypts (or encrypts) +// it using c in counter (CTR) mode with the initialization vector iv. +// The returned Reader does not buffer and has no block size. +// In CTR mode, encryption and decryption are the same operation: +// a CTR reader applied to an encrypted stream produces a decrypted +// stream and vice versa. +func NewCTRReader(c Cipher, iv []byte, r io.Reader) io.Reader { + return newXorReader(newCTRStream(c, iv), r) +} + +// NewCTRWriter returns a writer that encrypts (or decrypts) data using c +// in counter (CTR) mode with the initialization vector iv +// and writes the encrypted data to w. +// The returned Writer does not buffer and has no block size. +// In CTR mode, encryption and decryption are the same operation: +// a CTR writer applied to an decrypted stream produces an encrypted +// stream and vice versa. +func NewCTRWriter(c Cipher, iv []byte, w io.Writer) io.Writer { + return newXorWriter(newCTRStream(c, iv), w) +} diff --git a/libgo/go/crypto/block/eax.go b/libgo/go/crypto/block/eax.go new file mode 100644 index 000000000..3f3b96431 --- /dev/null +++ b/libgo/go/crypto/block/eax.go @@ -0,0 +1,253 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// EAX mode, not a NIST standard (yet). +// EAX provides encryption and authentication. +// EAX targets the same uses as NIST's CCM mode, +// but EAX adds the ability to run in streaming mode. + +// See +// http://csrc.nist.gov/groups/ST/toolkit/BCM/documents/proposedmodes/eax/eax-spec.pdf +// http://www.cs.ucdavis.edu/~rogaway/papers/eax.pdf +// What those papers call OMAC is now called CMAC. + +package block + +import ( + "fmt" + "hash" + "io" + "os" +) + +// An EAXTagError is returned when the message has failed to authenticate, +// because the tag at the end of the message stream (Read) does not match +// the tag computed from the message itself (Computed). +type EAXTagError struct { + Read []byte + Computed []byte +} + +func (e *EAXTagError) String() string { + return fmt.Sprintf("crypto/block: EAX tag mismatch: read %x but computed %x", e.Read, e.Computed) +} + +func setupEAX(c Cipher, iv, hdr []byte, tagBytes int) (ctrIV, tag []byte, cmac hash.Hash) { + n := len(iv) + if n != c.BlockSize() { + panic(fmt.Sprintln("crypto/block: EAX: iv length", n, "!=", c.BlockSize())) + } + buf := make([]byte, n) // zeroed + + // tag = CMAC(0 + iv) ^ CMAC(1 + hdr) ^ CMAC(2 + data) + cmac = NewCMAC(c) + cmac.Write(buf) // 0 + cmac.Write(iv) + sum := cmac.Sum() + ctrIV = dup(sum) + tag = dup(sum[0:tagBytes]) + + cmac.Reset() + buf[n-1] = 1 + cmac.Write(buf) // 1 + cmac.Write(hdr) + sum = cmac.Sum() + for i := 0; i < tagBytes; i++ { + tag[i] ^= sum[i] + } + + cmac.Reset() + buf[n-1] = 2 // 2 + cmac.Write(buf) + + return +} + +func finishEAX(tag []byte, cmac hash.Hash) { + // Finish CMAC #2 and xor into tag. + sum := cmac.Sum() + for i := range tag { + tag[i] ^= sum[i] + } +} + +// Writer adapter. Tees writes into both w and cmac. +// Knows that cmac never returns write errors. +type cmacWriter struct { + w io.Writer + cmac hash.Hash +} + +func (cw *cmacWriter) Write(p []byte) (n int, err os.Error) { + n, err = cw.w.Write(p) + cw.cmac.Write(p[0:n]) + return +} + +// An eaxEncrypter implements the EAX encryption mode. +type eaxEncrypter struct { + ctr io.Writer // CTR encrypter + cw cmacWriter // CTR's output stream + tag []byte +} + +// NewEAXEncrypter creates and returns a new EAX encrypter +// using the given cipher c, initialization vector iv, associated data hdr, +// and tag length tagBytes. The encrypter's Write method encrypts +// the data it receives and writes that data to w. +// The encrypter's Close method writes a final authenticating tag to w. +func NewEAXEncrypter(c Cipher, iv []byte, hdr []byte, tagBytes int, w io.Writer) io.WriteCloser { + x := new(eaxEncrypter) + + // Create new CTR instance writing to both + // w for encrypted output and cmac for digesting. + x.cw.w = w + var ctrIV []byte + ctrIV, x.tag, x.cw.cmac = setupEAX(c, iv, hdr, tagBytes) + x.ctr = NewCTRWriter(c, ctrIV, &x.cw) + return x +} + +func (x *eaxEncrypter) Write(p []byte) (n int, err os.Error) { + return x.ctr.Write(p) +} + +func (x *eaxEncrypter) Close() os.Error { + x.ctr = nil // crash if Write is called again + + // Write tag. + finishEAX(x.tag, x.cw.cmac) + n, err := x.cw.w.Write(x.tag) + if n != len(x.tag) && err == nil { + err = io.ErrShortWrite + } + + return err +} + +// Reader adapter. Returns data read from r but hangs +// on to the last len(tag) bytes for itself (returns EOF len(tag) +// bytes early). Also tees all data returned from Read into +// the cmac digest. The "don't return the last t bytes" +// and the "tee into digest" functionality could be separated, +// but the latter half is trivial. +type cmacReader struct { + r io.Reader + cmac hash.Hash + tag []byte + tmp []byte +} + +func (cr *cmacReader) Read(p []byte) (n int, err os.Error) { + // TODO(rsc): Maybe fall back to simpler code if + // we recognize the underlying r as a ByteBuffer + // or ByteReader. Then we can just take the last piece + // off at the start. + + // First, read a tag-sized chunk. + // It's probably not the tag (unless there's no data). + tag := cr.tag + if len(tag) < cap(tag) { + nt := len(tag) + nn, err1 := io.ReadFull(cr.r, tag[nt:cap(tag)]) + tag = tag[0 : nt+nn] + cr.tag = tag + if err1 != nil { + return 0, err1 + } + } + + tagBytes := len(tag) + if len(p) > 4*tagBytes { + // If p is big, try to read directly into p to avoid a copy. + n, err = cr.r.Read(p[tagBytes:]) + if n == 0 { + goto out + } + // copy old tag into p + for i := 0; i < tagBytes; i++ { + p[i] = tag[i] + } + // copy new tag out of p + for i := 0; i < tagBytes; i++ { + tag[i] = p[n+i] + } + goto out + } + + // Otherwise, read into p and then slide data + n, err = cr.r.Read(p) + if n == 0 { + goto out + } + + // copy tag+p into p+tmp and then swap tmp, tag + tmp := cr.tmp + for i := n + tagBytes - 1; i >= 0; i-- { + var c byte + if i < tagBytes { + c = tag[i] + } else { + c = p[i-tagBytes] + } + if i < n { + p[i] = c + } else { + tmp[i] = c + } + } + cr.tmp, cr.tag = tag, tmp + +out: + cr.cmac.Write(p[0:n]) + return +} + +type eaxDecrypter struct { + ctr io.Reader + cr cmacReader + tag []byte +} + +// NewEAXDecrypter creates and returns a new EAX decrypter +// using the given cipher c, initialization vector iv, associated data hdr, +// and tag length tagBytes. The encrypter's Read method decrypts and +// returns data read from r. At r's EOF, the encrypter checks the final +// authenticating tag and returns an EAXTagError if the tag is invalid. +// In that case, the message should be discarded. +// Note that the data stream returned from Read cannot be +// assumed to be valid, authenticated data until Read returns +// 0, nil to signal the end of the data. +func NewEAXDecrypter(c Cipher, iv []byte, hdr []byte, tagBytes int, r io.Reader) io.Reader { + x := new(eaxDecrypter) + + x.cr.r = r + x.cr.tag = make([]byte, 0, tagBytes) + x.cr.tmp = make([]byte, 0, tagBytes) + var ctrIV []byte + ctrIV, x.tag, x.cr.cmac = setupEAX(c, iv, hdr, tagBytes) + x.ctr = NewCTRReader(c, ctrIV, &x.cr) + return x +} + +func (x *eaxDecrypter) checkTag() os.Error { + x.ctr = nil // crash if Read is called again + + finishEAX(x.tag, x.cr.cmac) + if !same(x.tag, x.cr.tag) { + e := new(EAXTagError) + e.Computed = dup(x.tag) + e.Read = dup(x.cr.tag) + return e + } + return nil +} + +func (x *eaxDecrypter) Read(p []byte) (n int, err os.Error) { + n, err = x.ctr.Read(p) + if n == 0 && err == nil { + err = x.checkTag() + } + return n, err +} diff --git a/libgo/go/crypto/block/eax_aes_test.go b/libgo/go/crypto/block/eax_aes_test.go new file mode 100644 index 000000000..93aa771be --- /dev/null +++ b/libgo/go/crypto/block/eax_aes_test.go @@ -0,0 +1,140 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package block + +import ( + "bytes" + "crypto/aes" + "fmt" + "io" + "testing" +) + +// Test vectors from http://www.cs.ucdavis.edu/~rogaway/papers/eax.pdf + +type eaxAESTest struct { + msg []byte + key []byte + nonce []byte + header []byte + cipher []byte +} + +var eaxAESTests = []eaxAESTest{ + { + []byte{}, + []byte{0x23, 0x39, 0x52, 0xDE, 0xE4, 0xD5, 0xED, 0x5F, 0x9B, 0x9C, 0x6D, 0x6F, 0xF8, 0x0F, 0xF4, 0x78}, + []byte{0x62, 0xEC, 0x67, 0xF9, 0xC3, 0xA4, 0xA4, 0x07, 0xFC, 0xB2, 0xA8, 0xC4, 0x90, 0x31, 0xA8, 0xB3}, + []byte{0x6B, 0xFB, 0x91, 0x4F, 0xD0, 0x7E, 0xAE, 0x6B}, + []byte{0xE0, 0x37, 0x83, 0x0E, 0x83, 0x89, 0xF2, 0x7B, 0x02, 0x5A, 0x2D, 0x65, 0x27, 0xE7, 0x9D, 0x01}, + }, + { + []byte{0xF7, 0xFB}, + []byte{0x91, 0x94, 0x5D, 0x3F, 0x4D, 0xCB, 0xEE, 0x0B, 0xF4, 0x5E, 0xF5, 0x22, 0x55, 0xF0, 0x95, 0xA4}, + []byte{0xBE, 0xCA, 0xF0, 0x43, 0xB0, 0xA2, 0x3D, 0x84, 0x31, 0x94, 0xBA, 0x97, 0x2C, 0x66, 0xDE, 0xBD}, + []byte{0xFA, 0x3B, 0xFD, 0x48, 0x06, 0xEB, 0x53, 0xFA}, + []byte{0x19, 0xDD, 0x5C, 0x4C, 0x93, 0x31, 0x04, 0x9D, 0x0B, 0xDA, 0xB0, 0x27, 0x74, 0x08, 0xF6, 0x79, 0x67, 0xE5}, + }, + { + []byte{0x1A, 0x47, 0xCB, 0x49, 0x33}, + []byte{0x01, 0xF7, 0x4A, 0xD6, 0x40, 0x77, 0xF2, 0xE7, 0x04, 0xC0, 0xF6, 0x0A, 0xDA, 0x3D, 0xD5, 0x23}, + []byte{0x70, 0xC3, 0xDB, 0x4F, 0x0D, 0x26, 0x36, 0x84, 0x00, 0xA1, 0x0E, 0xD0, 0x5D, 0x2B, 0xFF, 0x5E}, + []byte{0x23, 0x4A, 0x34, 0x63, 0xC1, 0x26, 0x4A, 0xC6}, + []byte{0xD8, 0x51, 0xD5, 0xBA, 0xE0, 0x3A, 0x59, 0xF2, 0x38, 0xA2, 0x3E, 0x39, 0x19, 0x9D, 0xC9, 0x26, 0x66, 0x26, 0xC4, 0x0F, 0x80}, + }, + { + []byte{0x48, 0x1C, 0x9E, 0x39, 0xB1}, + []byte{0xD0, 0x7C, 0xF6, 0xCB, 0xB7, 0xF3, 0x13, 0xBD, 0xDE, 0x66, 0xB7, 0x27, 0xAF, 0xD3, 0xC5, 0xE8}, + []byte{0x84, 0x08, 0xDF, 0xFF, 0x3C, 0x1A, 0x2B, 0x12, 0x92, 0xDC, 0x19, 0x9E, 0x46, 0xB7, 0xD6, 0x17}, + []byte{0x33, 0xCC, 0xE2, 0xEA, 0xBF, 0xF5, 0xA7, 0x9D}, + []byte{0x63, 0x2A, 0x9D, 0x13, 0x1A, 0xD4, 0xC1, 0x68, 0xA4, 0x22, 0x5D, 0x8E, 0x1F, 0xF7, 0x55, 0x93, 0x99, 0x74, 0xA7, 0xBE, 0xDE}, + }, + { + []byte{0x40, 0xD0, 0xC0, 0x7D, 0xA5, 0xE4}, + []byte{0x35, 0xB6, 0xD0, 0x58, 0x00, 0x05, 0xBB, 0xC1, 0x2B, 0x05, 0x87, 0x12, 0x45, 0x57, 0xD2, 0xC2}, + []byte{0xFD, 0xB6, 0xB0, 0x66, 0x76, 0xEE, 0xDC, 0x5C, 0x61, 0xD7, 0x42, 0x76, 0xE1, 0xF8, 0xE8, 0x16}, + []byte{0xAE, 0xB9, 0x6E, 0xAE, 0xBE, 0x29, 0x70, 0xE9}, + []byte{0x07, 0x1D, 0xFE, 0x16, 0xC6, 0x75, 0xCB, 0x06, 0x77, 0xE5, 0x36, 0xF7, 0x3A, 0xFE, 0x6A, 0x14, 0xB7, 0x4E, 0xE4, 0x98, 0x44, 0xDD}, + }, + { + []byte{0x4D, 0xE3, 0xB3, 0x5C, 0x3F, 0xC0, 0x39, 0x24, 0x5B, 0xD1, 0xFB, 0x7D}, + []byte{0xBD, 0x8E, 0x6E, 0x11, 0x47, 0x5E, 0x60, 0xB2, 0x68, 0x78, 0x4C, 0x38, 0xC6, 0x2F, 0xEB, 0x22}, + []byte{0x6E, 0xAC, 0x5C, 0x93, 0x07, 0x2D, 0x8E, 0x85, 0x13, 0xF7, 0x50, 0x93, 0x5E, 0x46, 0xDA, 0x1B}, + []byte{0xD4, 0x48, 0x2D, 0x1C, 0xA7, 0x8D, 0xCE, 0x0F}, + []byte{0x83, 0x5B, 0xB4, 0xF1, 0x5D, 0x74, 0x3E, 0x35, 0x0E, 0x72, 0x84, 0x14, 0xAB, 0xB8, 0x64, 0x4F, 0xD6, 0xCC, 0xB8, 0x69, 0x47, 0xC5, 0xE1, 0x05, 0x90, 0x21, 0x0A, 0x4F}, + }, + { + []byte{0x8B, 0x0A, 0x79, 0x30, 0x6C, 0x9C, 0xE7, 0xED, 0x99, 0xDA, 0xE4, 0xF8, 0x7F, 0x8D, 0xD6, 0x16, 0x36}, + []byte{0x7C, 0x77, 0xD6, 0xE8, 0x13, 0xBE, 0xD5, 0xAC, 0x98, 0xBA, 0xA4, 0x17, 0x47, 0x7A, 0x2E, 0x7D}, + []byte{0x1A, 0x8C, 0x98, 0xDC, 0xD7, 0x3D, 0x38, 0x39, 0x3B, 0x2B, 0xF1, 0x56, 0x9D, 0xEE, 0xFC, 0x19}, + []byte{0x65, 0xD2, 0x01, 0x79, 0x90, 0xD6, 0x25, 0x28}, + []byte{0x02, 0x08, 0x3E, 0x39, 0x79, 0xDA, 0x01, 0x48, 0x12, 0xF5, 0x9F, 0x11, 0xD5, 0x26, 0x30, 0xDA, 0x30, 0x13, 0x73, 0x27, 0xD1, 0x06, 0x49, 0xB0, 0xAA, 0x6E, 0x1C, 0x18, 0x1D, 0xB6, 0x17, 0xD7, 0xF2}, + }, + { + []byte{0x1B, 0xDA, 0x12, 0x2B, 0xCE, 0x8A, 0x8D, 0xBA, 0xF1, 0x87, 0x7D, 0x96, 0x2B, 0x85, 0x92, 0xDD, 0x2D, 0x56}, + []byte{0x5F, 0xFF, 0x20, 0xCA, 0xFA, 0xB1, 0x19, 0xCA, 0x2F, 0xC7, 0x35, 0x49, 0xE2, 0x0F, 0x5B, 0x0D}, + []byte{0xDD, 0xE5, 0x9B, 0x97, 0xD7, 0x22, 0x15, 0x6D, 0x4D, 0x9A, 0xFF, 0x2B, 0xC7, 0x55, 0x98, 0x26}, + []byte{0x54, 0xB9, 0xF0, 0x4E, 0x6A, 0x09, 0x18, 0x9A}, + []byte{0x2E, 0xC4, 0x7B, 0x2C, 0x49, 0x54, 0xA4, 0x89, 0xAF, 0xC7, 0xBA, 0x48, 0x97, 0xED, 0xCD, 0xAE, 0x8C, 0xC3, 0x3B, 0x60, 0x45, 0x05, 0x99, 0xBD, 0x02, 0xC9, 0x63, 0x82, 0x90, 0x2A, 0xEF, 0x7F, 0x83, 0x2A}, + }, + { + []byte{0x6C, 0xF3, 0x67, 0x20, 0x87, 0x2B, 0x85, 0x13, 0xF6, 0xEA, 0xB1, 0xA8, 0xA4, 0x44, 0x38, 0xD5, 0xEF, 0x11}, + []byte{0xA4, 0xA4, 0x78, 0x2B, 0xCF, 0xFD, 0x3E, 0xC5, 0xE7, 0xEF, 0x6D, 0x8C, 0x34, 0xA5, 0x61, 0x23}, + []byte{0xB7, 0x81, 0xFC, 0xF2, 0xF7, 0x5F, 0xA5, 0xA8, 0xDE, 0x97, 0xA9, 0xCA, 0x48, 0xE5, 0x22, 0xEC}, + []byte{0x89, 0x9A, 0x17, 0x58, 0x97, 0x56, 0x1D, 0x7E}, + []byte{0x0D, 0xE1, 0x8F, 0xD0, 0xFD, 0xD9, 0x1E, 0x7A, 0xF1, 0x9F, 0x1D, 0x8E, 0xE8, 0x73, 0x39, 0x38, 0xB1, 0xE8, 0xE7, 0xF6, 0xD2, 0x23, 0x16, 0x18, 0x10, 0x2F, 0xDB, 0x7F, 0xE5, 0x5F, 0xF1, 0x99, 0x17, 0x00}, + }, + { + []byte{0xCA, 0x40, 0xD7, 0x44, 0x6E, 0x54, 0x5F, 0xFA, 0xED, 0x3B, 0xD1, 0x2A, 0x74, 0x0A, 0x65, 0x9F, 0xFB, 0xBB, 0x3C, 0xEA, 0xB7}, + []byte{0x83, 0x95, 0xFC, 0xF1, 0xE9, 0x5B, 0xEB, 0xD6, 0x97, 0xBD, 0x01, 0x0B, 0xC7, 0x66, 0xAA, 0xC3}, + []byte{0x22, 0xE7, 0xAD, 0xD9, 0x3C, 0xFC, 0x63, 0x93, 0xC5, 0x7E, 0xC0, 0xB3, 0xC1, 0x7D, 0x6B, 0x44}, + []byte{0x12, 0x67, 0x35, 0xFC, 0xC3, 0x20, 0xD2, 0x5A}, + []byte{0xCB, 0x89, 0x20, 0xF8, 0x7A, 0x6C, 0x75, 0xCF, 0xF3, 0x96, 0x27, 0xB5, 0x6E, 0x3E, 0xD1, 0x97, 0xC5, 0x52, 0xD2, 0x95, 0xA7, 0xCF, 0xC4, 0x6A, 0xFC, 0x25, 0x3B, 0x46, 0x52, 0xB1, 0xAF, 0x37, 0x95, 0xB1, 0x24, 0xAB, 0x6E}, + }, +} + +func TestEAXEncrypt_AES(t *testing.T) { + b := new(bytes.Buffer) + for i, tt := range eaxAESTests { + test := fmt.Sprintf("test %d", i) + c, err := aes.NewCipher(tt.key) + if err != nil { + t.Fatalf("%s: NewCipher(%d bytes) = %s", test, len(tt.key), err) + } + b.Reset() + enc := NewEAXEncrypter(c, tt.nonce, tt.header, 16, b) + n, err := io.Copy(enc, bytes.NewBuffer(tt.msg)) + if n != int64(len(tt.msg)) || err != nil { + t.Fatalf("%s: io.Copy into encrypter: %d, %s", test, n, err) + } + err = enc.Close() + if err != nil { + t.Fatalf("%s: enc.Close: %s", test, err) + } + if d := b.Bytes(); !same(d, tt.cipher) { + t.Fatalf("%s: got %x want %x", test, d, tt.cipher) + } + } +} + +func TestEAXDecrypt_AES(t *testing.T) { + b := new(bytes.Buffer) + for i, tt := range eaxAESTests { + test := fmt.Sprintf("test %d", i) + c, err := aes.NewCipher(tt.key) + if err != nil { + t.Fatalf("%s: NewCipher(%d bytes) = %s", test, len(tt.key), err) + } + b.Reset() + dec := NewEAXDecrypter(c, tt.nonce, tt.header, 16, bytes.NewBuffer(tt.cipher)) + n, err := io.Copy(b, dec) + if n != int64(len(tt.msg)) || err != nil { + t.Fatalf("%s: io.Copy into decrypter: %d, %s", test, n, err) + } + if d := b.Bytes(); !same(d, tt.msg) { + t.Fatalf("%s: got %x want %x", test, d, tt.msg) + } + } +} diff --git a/libgo/go/crypto/block/ecb.go b/libgo/go/crypto/block/ecb.go new file mode 100644 index 000000000..cf09f7cb3 --- /dev/null +++ b/libgo/go/crypto/block/ecb.go @@ -0,0 +1,270 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Electronic codebook (ECB) mode. +// ECB is a fancy name for ``encrypt and decrypt each block separately.'' +// It's a pretty bad thing to do for any large amount of data (more than one block), +// because the individual blocks can still be identified, duplicated, and reordered. +// The ECB implementation exists mainly to provide buffering for +// the other modes, which wrap it by providing modified Ciphers. + +// See NIST SP 800-38A, pp 9-10 + +package block + +import ( + "io" + "os" + "strconv" +) + +type ecbDecrypter struct { + c Cipher + r io.Reader + blockSize int // block size + + // Buffered data. + // The buffer buf is used as storage for both + // plain or crypt; at least one of those is nil at any given time. + buf []byte + plain []byte // plain text waiting to be read + crypt []byte // ciphertext waiting to be decrypted +} + +// Read into x.crypt until it has a full block or EOF or an error happens. +func (x *ecbDecrypter) fillCrypt() os.Error { + var err os.Error + for len(x.crypt) < x.blockSize { + off := len(x.crypt) + var m int + m, err = x.r.Read(x.crypt[off:x.blockSize]) + x.crypt = x.crypt[0 : off+m] + if m == 0 { + break + } + + // If an error happened but we got enough + // data to do some decryption, we can decrypt + // first and report the error (with some data) later. + // But if we don't have enough to decrypt, + // have to stop now. + if err != nil && len(x.crypt) < x.blockSize { + break + } + } + return err +} + +// Read from plain text buffer into p. +func (x *ecbDecrypter) readPlain(p []byte) int { + n := len(x.plain) + if n > len(p) { + n = len(p) + } + for i := 0; i < n; i++ { + p[i] = x.plain[i] + } + if n < len(x.plain) { + x.plain = x.plain[n:] + } else { + x.plain = nil + } + return n +} + +type ecbFragmentError int + +func (n ecbFragmentError) String() string { + return "crypto/block: " + strconv.Itoa(int(n)) + "-byte fragment at EOF" +} + +func (x *ecbDecrypter) Read(p []byte) (n int, err os.Error) { + if len(p) == 0 { + return + } + + // If there's no plaintext waiting and p is not big enough + // to hold a whole cipher block, we'll have to work in the + // cipher text buffer. Set it to non-nil so that the + // code below will fill it. + if x.plain == nil && len(p) < x.blockSize && x.crypt == nil { + x.crypt = x.buf[0:0] + } + + // If there is a leftover cipher text buffer, + // try to accumulate a full block. + if x.crypt != nil { + err = x.fillCrypt() + if err != nil || len(x.crypt) == 0 { + return + } + x.c.Decrypt(x.crypt, x.crypt) + x.plain = x.crypt + x.crypt = nil + } + + // If there is a leftover plain text buffer, read from it. + if x.plain != nil { + n = x.readPlain(p) + return + } + + // Read and decrypt directly in caller's buffer. + n, err = io.ReadAtLeast(x.r, p, x.blockSize) + if err == os.EOF && n > 0 { + // EOF is only okay on block boundary + err = os.ErrorString("block fragment at EOF during decryption") + return + } + var i int + for i = 0; i+x.blockSize <= n; i += x.blockSize { + a := p[i : i+x.blockSize] + x.c.Decrypt(a, a) + } + + // There might be an encrypted fringe remaining. + // Save it for next time. + if i < n { + p = p[i:n] + copy(x.buf, p) + x.crypt = x.buf[0:len(p)] + n = i + } + + return +} + +// NewECBDecrypter returns a reader that reads data from r and decrypts it using c. +// It decrypts by calling c.Decrypt on each block in sequence; +// this mode is known as electronic codebook mode, or ECB. +// The returned Reader does not buffer or read ahead except +// as required by the cipher's block size. +func NewECBDecrypter(c Cipher, r io.Reader) io.Reader { + x := new(ecbDecrypter) + x.c = c + x.r = r + x.blockSize = c.BlockSize() + x.buf = make([]byte, x.blockSize) + return x +} + +type ecbEncrypter struct { + c Cipher + w io.Writer + blockSize int + + // Buffered data. + // The buffer buf is used as storage for both + // plain or crypt. If both are non-nil, plain + // follows crypt in buf. + buf []byte + plain []byte // plain text waiting to be encrypted + crypt []byte // encrypted text waiting to be written +} + +// Flush the x.crypt buffer to x.w. +func (x *ecbEncrypter) flushCrypt() os.Error { + if len(x.crypt) == 0 { + return nil + } + n, err := x.w.Write(x.crypt) + if n < len(x.crypt) { + x.crypt = x.crypt[n:] + if err == nil { + err = io.ErrShortWrite + } + } + if err != nil { + return err + } + x.crypt = nil + return nil +} + +// Slide x.plain down to the beginning of x.buf. +// Plain is known to have less than one block of data, +// so this is cheap enough. +func (x *ecbEncrypter) slidePlain() { + if len(x.plain) == 0 { + x.plain = x.buf[0:0] + } else if cap(x.plain) < cap(x.buf) { + copy(x.buf, x.plain) + x.plain = x.buf[0:len(x.plain)] + } +} + +// Fill x.plain from the data in p. +// Return the number of bytes copied. +func (x *ecbEncrypter) fillPlain(p []byte) int { + off := len(x.plain) + n := len(p) + if max := cap(x.plain) - off; n > max { + n = max + } + x.plain = x.plain[0 : off+n] + for i := 0; i < n; i++ { + x.plain[off+i] = p[i] + } + return n +} + +// Encrypt x.plain; record encrypted range as x.crypt. +func (x *ecbEncrypter) encrypt() { + var i int + n := len(x.plain) + for i = 0; i+x.blockSize <= n; i += x.blockSize { + a := x.plain[i : i+x.blockSize] + x.c.Encrypt(a, a) + } + x.crypt = x.plain[0:i] + x.plain = x.plain[i:n] +} + +func (x *ecbEncrypter) Write(p []byte) (n int, err os.Error) { + for { + // If there is data waiting to be written, write it. + // This can happen on the first iteration + // if a write failed in an earlier call. + if err = x.flushCrypt(); err != nil { + return + } + + // Now that encrypted data is gone (flush ran), + // perhaps we need to slide the plaintext down. + x.slidePlain() + + // Fill plaintext buffer from p. + m := x.fillPlain(p) + if m == 0 { + break + } + n += m + p = p[m:] + + // Encrypt, adjusting crypt and plain. + x.encrypt() + + // Write x.crypt. + if err = x.flushCrypt(); err != nil { + break + } + } + return +} + +// NewECBEncrypter returns a writer that encrypts data using c and writes it to w. +// It encrypts by calling c.Encrypt on each block in sequence; +// this mode is known as electronic codebook mode, or ECB. +// The returned Writer does no buffering except as required +// by the cipher's block size, so there is no need for a Flush method. +func NewECBEncrypter(c Cipher, w io.Writer) io.Writer { + x := new(ecbEncrypter) + x.c = c + x.w = w + x.blockSize = c.BlockSize() + + // Create a buffer that is an integral number of blocks. + x.buf = make([]byte, 8192/x.blockSize*x.blockSize) + return x +} diff --git a/libgo/go/crypto/block/ecb_aes_test.go b/libgo/go/crypto/block/ecb_aes_test.go new file mode 100644 index 000000000..14481d096 --- /dev/null +++ b/libgo/go/crypto/block/ecb_aes_test.go @@ -0,0 +1,127 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// ECB AES test vectors. + +// See U.S. National Institute of Standards and Technology (NIST) +// Special Publication 800-38A, ``Recommendation for Block Cipher +// Modes of Operation,'' 2001 Edition, pp. 24-27. + +package block + +import ( + "bytes" + "crypto/aes" + "io" + "testing" +) + +type ecbTest struct { + name string + key []byte + in []byte + out []byte +} + +var commonInput = []byte{ + 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a, + 0xae, 0x2d, 0x8a, 0x57, 0x1e, 0x03, 0xac, 0x9c, 0x9e, 0xb7, 0x6f, 0xac, 0x45, 0xaf, 0x8e, 0x51, + 0x30, 0xc8, 0x1c, 0x46, 0xa3, 0x5c, 0xe4, 0x11, 0xe5, 0xfb, 0xc1, 0x19, 0x1a, 0x0a, 0x52, 0xef, + 0xf6, 0x9f, 0x24, 0x45, 0xdf, 0x4f, 0x9b, 0x17, 0xad, 0x2b, 0x41, 0x7b, 0xe6, 0x6c, 0x37, 0x10, +} + +var commonKey128 = []byte{0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c} + +var commonKey192 = []byte{ + 0x8e, 0x73, 0xb0, 0xf7, 0xda, 0x0e, 0x64, 0x52, 0xc8, 0x10, 0xf3, 0x2b, 0x80, 0x90, 0x79, 0xe5, + 0x62, 0xf8, 0xea, 0xd2, 0x52, 0x2c, 0x6b, 0x7b, +} + +var commonKey256 = []byte{ + 0x60, 0x3d, 0xeb, 0x10, 0x15, 0xca, 0x71, 0xbe, 0x2b, 0x73, 0xae, 0xf0, 0x85, 0x7d, 0x77, 0x81, + 0x1f, 0x35, 0x2c, 0x07, 0x3b, 0x61, 0x08, 0xd7, 0x2d, 0x98, 0x10, 0xa3, 0x09, 0x14, 0xdf, 0xf4, +} + +var commonIV = []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f} + +var ecbAESTests = []ecbTest{ + // FIPS 197, Appendix B, C + { + "FIPS-197 Appendix B", + commonKey128, + []byte{0x32, 0x43, 0xf6, 0xa8, 0x88, 0x5a, 0x30, 0x8d, 0x31, 0x31, 0x98, 0xa2, 0xe0, 0x37, 0x07, 0x34}, + []byte{0x39, 0x25, 0x84, 0x1d, 0x02, 0xdc, 0x09, 0xfb, 0xdc, 0x11, 0x85, 0x97, 0x19, 0x6a, 0x0b, 0x32}, + }, + + // NIST SP 800-38A pp 24-27 + { + "ECB-AES128", + commonKey128, + commonInput, + []byte{ + 0x3a, 0xd7, 0x7b, 0xb4, 0x0d, 0x7a, 0x36, 0x60, 0xa8, 0x9e, 0xca, 0xf3, 0x24, 0x66, 0xef, 0x97, + 0xf5, 0xd3, 0xd5, 0x85, 0x03, 0xb9, 0x69, 0x9d, 0xe7, 0x85, 0x89, 0x5a, 0x96, 0xfd, 0xba, 0xaf, + 0x43, 0xb1, 0xcd, 0x7f, 0x59, 0x8e, 0xce, 0x23, 0x88, 0x1b, 0x00, 0xe3, 0xed, 0x03, 0x06, 0x88, + 0x7b, 0x0c, 0x78, 0x5e, 0x27, 0xe8, 0xad, 0x3f, 0x82, 0x23, 0x20, 0x71, 0x04, 0x72, 0x5d, 0xd4, + }, + }, + { + "ECB-AES192", + commonKey192, + commonInput, + []byte{ + 0xbd, 0x33, 0x4f, 0x1d, 0x6e, 0x45, 0xf2, 0x5f, 0xf7, 0x12, 0xa2, 0x14, 0x57, 0x1f, 0xa5, 0xcc, + 0x97, 0x41, 0x04, 0x84, 0x6d, 0x0a, 0xd3, 0xad, 0x77, 0x34, 0xec, 0xb3, 0xec, 0xee, 0x4e, 0xef, + 0xef, 0x7a, 0xfd, 0x22, 0x70, 0xe2, 0xe6, 0x0a, 0xdc, 0xe0, 0xba, 0x2f, 0xac, 0xe6, 0x44, 0x4e, + 0x9a, 0x4b, 0x41, 0xba, 0x73, 0x8d, 0x6c, 0x72, 0xfb, 0x16, 0x69, 0x16, 0x03, 0xc1, 0x8e, 0x0e, + }, + }, + { + "ECB-AES256", + commonKey256, + commonInput, + []byte{ + 0xf3, 0xee, 0xd1, 0xbd, 0xb5, 0xd2, 0xa0, 0x3c, 0x06, 0x4b, 0x5a, 0x7e, 0x3d, 0xb1, 0x81, 0xf8, + 0x59, 0x1c, 0xcb, 0x10, 0xd4, 0x10, 0xed, 0x26, 0xdc, 0x5b, 0xa7, 0x4a, 0x31, 0x36, 0x28, 0x70, + 0xb6, 0xed, 0x21, 0xb9, 0x9c, 0xa6, 0xf4, 0xf9, 0xf1, 0x53, 0xe7, 0xb1, 0xbe, 0xaf, 0xed, 0x1d, + 0x23, 0x30, 0x4b, 0x7a, 0x39, 0xf9, 0xf3, 0xff, 0x06, 0x7d, 0x8d, 0x8f, 0x9e, 0x24, 0xec, 0xc7, + }, + }, +} + +func TestECB_AES(t *testing.T) { + for _, tt := range ecbAESTests { + test := tt.name + + c, err := aes.NewCipher(tt.key) + if err != nil { + t.Errorf("%s: NewCipher(%d bytes) = %s", test, len(tt.key), err) + continue + } + + var crypt bytes.Buffer + w := NewECBEncrypter(c, &crypt) + var r io.Reader = bytes.NewBuffer(tt.in) + n, err := io.Copy(w, r) + if n != int64(len(tt.in)) || err != nil { + t.Errorf("%s: ECBReader io.Copy = %d, %v want %d, nil", test, n, err, len(tt.in)) + } else if d := crypt.Bytes(); !same(tt.out, d) { + t.Errorf("%s: ECBReader\nhave %x\nwant %x", test, d, tt.out) + } + + var plain bytes.Buffer + r = NewECBDecrypter(c, bytes.NewBuffer(tt.out)) + w = &plain + n, err = io.Copy(w, r) + if n != int64(len(tt.out)) || err != nil { + t.Errorf("%s: ECBWriter io.Copy = %d, %v want %d, nil", test, n, err, len(tt.out)) + } else if d := plain.Bytes(); !same(tt.in, d) { + t.Errorf("%s: ECBWriter\nhave %x\nwant %x", test, d, tt.in) + } + + if t.Failed() { + break + } + } +} diff --git a/libgo/go/crypto/block/ecb_test.go b/libgo/go/crypto/block/ecb_test.go new file mode 100644 index 000000000..6f79d929a --- /dev/null +++ b/libgo/go/crypto/block/ecb_test.go @@ -0,0 +1,181 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package block + +import ( + "bytes" + "fmt" + "io" + "testing" + "testing/iotest" +) + +// Simple Cipher for testing: adds an incrementing amount +// to each byte in each +type IncCipher struct { + blockSize int + delta byte + encrypting bool +} + +func (c *IncCipher) BlockSize() int { return c.blockSize } + +func (c *IncCipher) Encrypt(dst, src []byte) { + if !c.encrypting { + panic("encrypt: not encrypting") + } + if len(src) != c.blockSize || len(dst) != c.blockSize { + panic(fmt.Sprintln("encrypt: wrong block size", c.blockSize, len(src), len(dst))) + } + c.delta++ + for i, b := range src { + dst[i] = b + c.delta + } +} + +func (c *IncCipher) Decrypt(dst, src []byte) { + if c.encrypting { + panic("decrypt: not decrypting") + } + if len(src) != c.blockSize || len(dst) != c.blockSize { + panic(fmt.Sprintln("decrypt: wrong block size ", c.blockSize, " ", len(src), " ", len(dst))) + } + c.delta-- + for i, b := range src { + dst[i] = b + c.delta + } +} + +func TestECBEncrypter(t *testing.T) { + var plain, crypt [256]byte + for i := 0; i < len(plain); i++ { + plain[i] = byte(i) + } + b := new(bytes.Buffer) + for block := 1; block <= 64; block *= 2 { + // compute encrypted version + delta := byte(0) + for i := 0; i < len(crypt); i++ { + if i%block == 0 { + delta++ + } + crypt[i] = plain[i] + delta + } + + for frag := 0; frag < 2; frag++ { + c := &IncCipher{block, 0, true} + b.Reset() + r := bytes.NewBuffer(plain[0:]) + w := NewECBEncrypter(c, b) + + // copy plain into w in increasingly large chunks: 1, 1, 2, 4, 8, ... + // if frag != 0, move the 1 to the end to cause fragmentation. + if frag == 0 { + _, err := io.Copyn(w, r, 1) + if err != nil { + t.Errorf("block=%d frag=0: first Copyn: %s", block, err) + continue + } + } + for n := 1; n <= len(plain)/2; n *= 2 { + _, err := io.Copyn(w, r, int64(n)) + if err != nil { + t.Errorf("block=%d frag=%d: Copyn %d: %s", block, frag, n, err) + } + } + if frag != 0 { + _, err := io.Copyn(w, r, 1) + if err != nil { + t.Errorf("block=%d frag=1: last Copyn: %s", block, err) + continue + } + } + + // check output + data := b.Bytes() + if len(data) != len(crypt) { + t.Errorf("block=%d frag=%d: want %d bytes, got %d", block, frag, len(crypt), len(data)) + continue + } + + if string(data) != string(crypt[0:]) { + t.Errorf("block=%d frag=%d: want %x got %x", block, frag, data, crypt) + } + } + } +} + +func testECBDecrypter(t *testing.T, maxio int) { + var readers = []func(io.Reader) io.Reader{ + func(r io.Reader) io.Reader { return r }, + iotest.OneByteReader, + iotest.HalfReader, + } + var plain, crypt [256]byte + for i := 0; i < len(plain); i++ { + plain[i] = byte(255 - i) + } + b := new(bytes.Buffer) + for block := 1; block <= 64 && block <= maxio; block *= 2 { + // compute encrypted version + delta := byte(0) + for i := 0; i < len(crypt); i++ { + if i%block == 0 { + delta++ + } + crypt[i] = plain[i] + delta + } + + for mode := 0; mode < len(readers); mode++ { + for frag := 0; frag < 2; frag++ { + test := fmt.Sprintf("block=%d mode=%d frag=%d maxio=%d", block, mode, frag, maxio) + c := &IncCipher{block, 0, false} + b.Reset() + r := NewECBDecrypter(c, readers[mode](bytes.NewBuffer(crypt[0:maxio]))) + + // read from crypt in increasingly large chunks: 1, 1, 2, 4, 8, ... + // if frag == 1, move the 1 to the end to cause fragmentation. + if frag == 0 { + _, err := io.Copyn(b, r, 1) + if err != nil { + t.Errorf("%s: first Copyn: %s", test, err) + continue + } + } + for n := 1; n <= maxio/2; n *= 2 { + _, err := io.Copyn(b, r, int64(n)) + if err != nil { + t.Errorf("%s: Copyn %d: %s", test, n, err) + } + } + if frag != 0 { + _, err := io.Copyn(b, r, 1) + if err != nil { + t.Errorf("%s: last Copyn: %s", test, err) + continue + } + } + + // check output + data := b.Bytes() + if len(data) != maxio { + t.Errorf("%s: want %d bytes, got %d", test, maxio, len(data)) + continue + } + + if string(data) != string(plain[0:maxio]) { + t.Errorf("%s: input=%x want %x got %x", test, crypt[0:maxio], plain[0:maxio], data) + } + } + } + } +} + +func TestECBDecrypter(t *testing.T) { + // Do shorter I/O sizes first; they're easier to debug. + for n := 1; n <= 256 && !t.Failed(); n *= 2 { + testECBDecrypter(t, n) + } +} diff --git a/libgo/go/crypto/block/ofb.go b/libgo/go/crypto/block/ofb.go new file mode 100644 index 000000000..11aaaa4d7 --- /dev/null +++ b/libgo/go/crypto/block/ofb.go @@ -0,0 +1,60 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Output feedback (OFB) mode. + +// OFB converts a block cipher into a stream cipher by +// repeatedly encrypting an initialization vector and +// xoring the resulting stream of data with the input. + +// See NIST SP 800-38A, pp 13-15 + +package block + +import ( + "fmt" + "io" +) + +type ofbStream struct { + c Cipher + iv []byte +} + +func newOFBStream(c Cipher, iv []byte) *ofbStream { + x := new(ofbStream) + x.c = c + n := len(iv) + if n != c.BlockSize() { + panic(fmt.Sprintln("crypto/block: newOFBStream: invalid iv size", n, "!=", c.BlockSize())) + } + x.iv = dup(iv) + return x +} + +func (x *ofbStream) Next() []byte { + x.c.Encrypt(x.iv, x.iv) + return x.iv +} + +// NewOFBReader returns a reader that reads data from r, decrypts (or encrypts) +// it using c in output feedback (OFB) mode with the initialization vector iv. +// The returned Reader does not buffer and has no block size. +// In OFB mode, encryption and decryption are the same operation: +// an OFB reader applied to an encrypted stream produces a decrypted +// stream and vice versa. +func NewOFBReader(c Cipher, iv []byte, r io.Reader) io.Reader { + return newXorReader(newOFBStream(c, iv), r) +} + +// NewOFBWriter returns a writer that encrypts (or decrypts) data using c +// in cipher feedback (OFB) mode with the initialization vector iv +// and writes the encrypted data to w. +// The returned Writer does not buffer and has no block size. +// In OFB mode, encryption and decryption are the same operation: +// an OFB writer applied to an decrypted stream produces an encrypted +// stream and vice versa. +func NewOFBWriter(c Cipher, iv []byte, w io.Writer) io.Writer { + return newXorWriter(newOFBStream(c, iv), w) +} diff --git a/libgo/go/crypto/block/ofb_aes_test.go b/libgo/go/crypto/block/ofb_aes_test.go new file mode 100644 index 000000000..9c527a6b3 --- /dev/null +++ b/libgo/go/crypto/block/ofb_aes_test.go @@ -0,0 +1,108 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// OFB AES test vectors. + +// See U.S. National Institute of Standards and Technology (NIST) +// Special Publication 800-38A, ``Recommendation for Block Cipher +// Modes of Operation,'' 2001 Edition, pp. 52-55. + +package block + +import ( + "bytes" + "crypto/aes" + "io" + "testing" +) + +type ofbTest struct { + name string + key []byte + iv []byte + in []byte + out []byte +} + +var ofbAESTests = []ofbTest{ + // NIST SP 800-38A pp 52-55 + { + "OFB-AES128", + commonKey128, + commonIV, + commonInput, + []byte{ + 0x3b, 0x3f, 0xd9, 0x2e, 0xb7, 0x2d, 0xad, 0x20, 0x33, 0x34, 0x49, 0xf8, 0xe8, 0x3c, 0xfb, 0x4a, + 0x77, 0x89, 0x50, 0x8d, 0x16, 0x91, 0x8f, 0x03, 0xf5, 0x3c, 0x52, 0xda, 0xc5, 0x4e, 0xd8, 0x25, + 0x97, 0x40, 0x05, 0x1e, 0x9c, 0x5f, 0xec, 0xf6, 0x43, 0x44, 0xf7, 0xa8, 0x22, 0x60, 0xed, 0xcc, + 0x30, 0x4c, 0x65, 0x28, 0xf6, 0x59, 0xc7, 0x78, 0x66, 0xa5, 0x10, 0xd9, 0xc1, 0xd6, 0xae, 0x5e, + }, + }, + { + "OFB-AES192", + commonKey192, + commonIV, + commonInput, + []byte{ + 0xcd, 0xc8, 0x0d, 0x6f, 0xdd, 0xf1, 0x8c, 0xab, 0x34, 0xc2, 0x59, 0x09, 0xc9, 0x9a, 0x41, 0x74, + 0xfc, 0xc2, 0x8b, 0x8d, 0x4c, 0x63, 0x83, 0x7c, 0x09, 0xe8, 0x17, 0x00, 0xc1, 0x10, 0x04, 0x01, + 0x8d, 0x9a, 0x9a, 0xea, 0xc0, 0xf6, 0x59, 0x6f, 0x55, 0x9c, 0x6d, 0x4d, 0xaf, 0x59, 0xa5, 0xf2, + 0x6d, 0x9f, 0x20, 0x08, 0x57, 0xca, 0x6c, 0x3e, 0x9c, 0xac, 0x52, 0x4b, 0xd9, 0xac, 0xc9, 0x2a, + }, + }, + { + "OFB-AES256", + commonKey256, + commonIV, + commonInput, + []byte{ + 0xdc, 0x7e, 0x84, 0xbf, 0xda, 0x79, 0x16, 0x4b, 0x7e, 0xcd, 0x84, 0x86, 0x98, 0x5d, 0x38, 0x60, + 0x4f, 0xeb, 0xdc, 0x67, 0x40, 0xd2, 0x0b, 0x3a, 0xc8, 0x8f, 0x6a, 0xd8, 0x2a, 0x4f, 0xb0, 0x8d, + 0x71, 0xab, 0x47, 0xa0, 0x86, 0xe8, 0x6e, 0xed, 0xf3, 0x9d, 0x1c, 0x5b, 0xba, 0x97, 0xc4, 0x08, + 0x01, 0x26, 0x14, 0x1d, 0x67, 0xf3, 0x7b, 0xe8, 0x53, 0x8f, 0x5a, 0x8b, 0xe7, 0x40, 0xe4, 0x84, + }, + }, +} + +func TestOFB_AES(t *testing.T) { + for _, tt := range ofbAESTests { + test := tt.name + + c, err := aes.NewCipher(tt.key) + if err != nil { + t.Errorf("%s: NewCipher(%d bytes) = %s", test, len(tt.key), err) + continue + } + + for j := 0; j <= 5; j += 5 { + var crypt bytes.Buffer + in := tt.in[0 : len(tt.in)-j] + w := NewOFBWriter(c, tt.iv, &crypt) + var r io.Reader = bytes.NewBuffer(in) + n, err := io.Copy(w, r) + if n != int64(len(in)) || err != nil { + t.Errorf("%s/%d: OFBWriter io.Copy = %d, %v want %d, nil", test, len(in), n, err, len(in)) + } else if d, out := crypt.Bytes(), tt.out[0:len(in)]; !same(out, d) { + t.Errorf("%s/%d: OFBWriter\ninpt %x\nhave %x\nwant %x", test, len(in), in, d, out) + } + } + + for j := 0; j <= 7; j += 7 { + var plain bytes.Buffer + out := tt.out[0 : len(tt.out)-j] + r := NewOFBReader(c, tt.iv, bytes.NewBuffer(out)) + w := &plain + n, err := io.Copy(w, r) + if n != int64(len(out)) || err != nil { + t.Errorf("%s/%d: OFBReader io.Copy = %d, %v want %d, nil", test, len(out), n, err, len(out)) + } else if d, in := plain.Bytes(), tt.in[0:len(out)]; !same(in, d) { + t.Errorf("%s/%d: OFBReader\nhave %x\nwant %x", test, len(out), d, in) + } + } + + if t.Failed() { + break + } + } +} diff --git a/libgo/go/crypto/block/xor.go b/libgo/go/crypto/block/xor.go new file mode 100644 index 000000000..9d8b17224 --- /dev/null +++ b/libgo/go/crypto/block/xor.go @@ -0,0 +1,124 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Encrypt/decrypt data by xor with a pseudo-random data stream. + +package block + +import ( + "io" + "os" +) + +// A dataStream is an interface to an unending stream of data, +// used by XorReader and XorWriter to model a pseudo-random generator. +// Calls to Next() return sequential blocks of data from the stream. +// Each call must return at least one byte: there is no EOF. +type dataStream interface { + Next() []byte +} + +type xorReader struct { + r io.Reader + rand dataStream // pseudo-random + buf []byte // data available from last call to rand +} + +func newXorReader(rand dataStream, r io.Reader) io.Reader { + x := new(xorReader) + x.r = r + x.rand = rand + return x +} + +func (x *xorReader) Read(p []byte) (n int, err os.Error) { + n, err = x.r.Read(p) + + // xor input with stream. + bp := 0 + buf := x.buf + for i := 0; i < n; i++ { + if bp >= len(buf) { + buf = x.rand.Next() + bp = 0 + } + p[i] ^= buf[bp] + bp++ + } + x.buf = buf[bp:] + return n, err +} + +type xorWriter struct { + w io.Writer + rand dataStream // pseudo-random + buf []byte // last buffer returned by rand + extra []byte // extra random data (use before buf) + work []byte // work space +} + +func newXorWriter(rand dataStream, w io.Writer) io.Writer { + x := new(xorWriter) + x.w = w + x.rand = rand + x.work = make([]byte, 4096) + return x +} + +func (x *xorWriter) Write(p []byte) (n int, err os.Error) { + for len(p) > 0 { + // Determine next chunk of random data + // and xor with p into x.work. + var chunk []byte + m := len(p) + if nn := len(x.extra); nn > 0 { + // extra points into work, so edit directly + if m > nn { + m = nn + } + for i := 0; i < m; i++ { + x.extra[i] ^= p[i] + } + chunk = x.extra[0:m] + } else { + // xor p ^ buf into work, refreshing buf as needed + if nn := len(x.work); m > nn { + m = nn + } + bp := 0 + buf := x.buf + for i := 0; i < m; i++ { + if bp >= len(buf) { + buf = x.rand.Next() + bp = 0 + } + x.work[i] = buf[bp] ^ p[i] + bp++ + } + x.buf = buf[bp:] + chunk = x.work[0:m] + } + + // Write chunk. + var nn int + nn, err = x.w.Write(chunk) + if nn != len(chunk) && err == nil { + err = io.ErrShortWrite + } + if nn < len(chunk) { + // Reconstruct the random bits from the unwritten + // data and save them for next time. + for i := nn; i < m; i++ { + chunk[i] ^= p[i] + } + x.extra = chunk[nn:] + } + n += nn + if err != nil { + return + } + p = p[m:] + } + return +} diff --git a/libgo/go/crypto/block/xor_test.go b/libgo/go/crypto/block/xor_test.go new file mode 100644 index 000000000..50f6bb08d --- /dev/null +++ b/libgo/go/crypto/block/xor_test.go @@ -0,0 +1,168 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package block + +import ( + "bytes" + "fmt" + "io" + "testing" + "testing/iotest" +) + +// Simple "pseudo-random" stream for testing. +type incStream struct { + buf []byte + n byte +} + +func newIncStream(blockSize int) *incStream { + x := new(incStream) + x.buf = make([]byte, blockSize) + return x +} + +func (x *incStream) Next() []byte { + x.n++ + for i := range x.buf { + x.buf[i] = x.n + x.n++ + } + return x.buf +} + +func testXorWriter(t *testing.T, maxio int) { + var plain, crypt [256]byte + for i := 0; i < len(plain); i++ { + plain[i] = byte(i) + } + b := new(bytes.Buffer) + for block := 1; block <= 64 && block <= maxio; block *= 2 { + // compute encrypted version + n := byte(0) + for i := 0; i < len(crypt); i++ { + if i%block == 0 { + n++ + } + crypt[i] = plain[i] ^ n + n++ + } + + for frag := 0; frag < 2; frag++ { + test := fmt.Sprintf("block=%d frag=%d maxio=%d", block, frag, maxio) + b.Reset() + r := bytes.NewBuffer(plain[0:]) + s := newIncStream(block) + w := newXorWriter(s, b) + + // copy plain into w in increasingly large chunks: 1, 1, 2, 4, 8, ... + // if frag != 0, move the 1 to the end to cause fragmentation. + if frag == 0 { + _, err := io.Copyn(w, r, 1) + if err != nil { + t.Errorf("%s: first Copyn: %s", test, err) + continue + } + } + for n := 1; n <= len(plain)/2; n *= 2 { + _, err := io.Copyn(w, r, int64(n)) + if err != nil { + t.Errorf("%s: Copyn %d: %s", test, n, err) + } + } + + // check output + crypt := crypt[0 : len(crypt)-frag] + data := b.Bytes() + if len(data) != len(crypt) { + t.Errorf("%s: want %d bytes, got %d", test, len(crypt), len(data)) + continue + } + + if string(data) != string(crypt) { + t.Errorf("%s: want %x got %x", test, data, crypt) + } + } + } +} + + +func TestXorWriter(t *testing.T) { + // Do shorter I/O sizes first; they're easier to debug. + for n := 1; n <= 256 && !t.Failed(); n *= 2 { + testXorWriter(t, n) + } +} + +func testXorReader(t *testing.T, maxio int) { + var readers = []func(io.Reader) io.Reader{ + func(r io.Reader) io.Reader { return r }, + iotest.OneByteReader, + iotest.HalfReader, + } + var plain, crypt [256]byte + for i := 0; i < len(plain); i++ { + plain[i] = byte(255 - i) + } + b := new(bytes.Buffer) + for block := 1; block <= 64 && block <= maxio; block *= 2 { + // compute encrypted version + n := byte(0) + for i := 0; i < len(crypt); i++ { + if i%block == 0 { + n++ + } + crypt[i] = plain[i] ^ n + n++ + } + + for mode := 0; mode < len(readers); mode++ { + for frag := 0; frag < 2; frag++ { + test := fmt.Sprintf("block=%d mode=%d frag=%d maxio=%d", block, mode, frag, maxio) + s := newIncStream(block) + b.Reset() + r := newXorReader(s, readers[mode](bytes.NewBuffer(crypt[0:maxio]))) + + // read from crypt in increasingly large chunks: 1, 1, 2, 4, 8, ... + // if frag == 1, move the 1 to the end to cause fragmentation. + if frag == 0 { + _, err := io.Copyn(b, r, 1) + if err != nil { + t.Errorf("%s: first Copyn: %s", test, err) + continue + } + } + for n := 1; n <= maxio/2; n *= 2 { + _, err := io.Copyn(b, r, int64(n)) + if err != nil { + t.Errorf("%s: Copyn %d: %s", test, n, err) + } + } + + // check output + data := b.Bytes() + crypt := crypt[0 : maxio-frag] + plain := plain[0 : maxio-frag] + if len(data) != len(plain) { + t.Errorf("%s: want %d bytes, got %d", test, len(plain), len(data)) + continue + } + + if string(data) != string(plain) { + t.Errorf("%s: input=%x want %x got %x", test, crypt, plain, data) + } + } + } + } +} + +func TestXorReader(t *testing.T) { + // Do shorter I/O sizes first; they're easier to debug. + for n := 1; n <= 256 && !t.Failed(); n *= 2 { + testXorReader(t, n) + } +} + +// TODO(rsc): Test handling of writes after write errors. diff --git a/libgo/go/crypto/blowfish/block.go b/libgo/go/crypto/blowfish/block.go new file mode 100644 index 000000000..7fbe7eefb --- /dev/null +++ b/libgo/go/crypto/blowfish/block.go @@ -0,0 +1,101 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package blowfish + +func expandKey(key []byte, c *Cipher) { + copy(c.p[0:], p[0:]) + copy(c.s0[0:], s0[0:]) + copy(c.s1[0:], s1[0:]) + copy(c.s2[0:], s2[0:]) + copy(c.s3[0:], s3[0:]) + + j := 0 + for i := 0; i < 18; i++ { + var d uint32 + for k := 0; k < 4; k++ { + d = d<<8 | uint32(key[j])&0x000000FF + j++ + if j >= len(key) { + j = 0 + } + } + c.p[i] ^= d + } + + var l, r uint32 + for i := 0; i < 18; i += 2 { + l, r = encryptBlock(l, r, c) + c.p[i], c.p[i+1] = l, r + } + + for i := 0; i < 256; i += 2 { + l, r = encryptBlock(l, r, c) + c.s0[i], c.s0[i+1] = l, r + } + for i := 0; i < 256; i += 2 { + l, r = encryptBlock(l, r, c) + c.s1[i], c.s1[i+1] = l, r + } + for i := 0; i < 256; i += 2 { + l, r = encryptBlock(l, r, c) + c.s2[i], c.s2[i+1] = l, r + } + for i := 0; i < 256; i += 2 { + l, r = encryptBlock(l, r, c) + c.s3[i], c.s3[i+1] = l, r + } +} + +func encryptBlock(l, r uint32, c *Cipher) (uint32, uint32) { + xl, xr := l, r + xl ^= c.p[0] + xr ^= ((c.s0[byte(xl>>24)] + c.s1[byte(xl>>16)]) ^ c.s2[byte(xl>>8)]) + c.s3[byte(xl)] ^ c.p[1] + xl ^= ((c.s0[byte(xr>>24)] + c.s1[byte(xr>>16)]) ^ c.s2[byte(xr>>8)]) + c.s3[byte(xr)] ^ c.p[2] + xr ^= ((c.s0[byte(xl>>24)] + c.s1[byte(xl>>16)]) ^ c.s2[byte(xl>>8)]) + c.s3[byte(xl)] ^ c.p[3] + xl ^= ((c.s0[byte(xr>>24)] + c.s1[byte(xr>>16)]) ^ c.s2[byte(xr>>8)]) + c.s3[byte(xr)] ^ c.p[4] + xr ^= ((c.s0[byte(xl>>24)] + c.s1[byte(xl>>16)]) ^ c.s2[byte(xl>>8)]) + c.s3[byte(xl)] ^ c.p[5] + xl ^= ((c.s0[byte(xr>>24)] + c.s1[byte(xr>>16)]) ^ c.s2[byte(xr>>8)]) + c.s3[byte(xr)] ^ c.p[6] + xr ^= ((c.s0[byte(xl>>24)] + c.s1[byte(xl>>16)]) ^ c.s2[byte(xl>>8)]) + c.s3[byte(xl)] ^ c.p[7] + xl ^= ((c.s0[byte(xr>>24)] + c.s1[byte(xr>>16)]) ^ c.s2[byte(xr>>8)]) + c.s3[byte(xr)] ^ c.p[8] + xr ^= ((c.s0[byte(xl>>24)] + c.s1[byte(xl>>16)]) ^ c.s2[byte(xl>>8)]) + c.s3[byte(xl)] ^ c.p[9] + xl ^= ((c.s0[byte(xr>>24)] + c.s1[byte(xr>>16)]) ^ c.s2[byte(xr>>8)]) + c.s3[byte(xr)] ^ c.p[10] + xr ^= ((c.s0[byte(xl>>24)] + c.s1[byte(xl>>16)]) ^ c.s2[byte(xl>>8)]) + c.s3[byte(xl)] ^ c.p[11] + xl ^= ((c.s0[byte(xr>>24)] + c.s1[byte(xr>>16)]) ^ c.s2[byte(xr>>8)]) + c.s3[byte(xr)] ^ c.p[12] + xr ^= ((c.s0[byte(xl>>24)] + c.s1[byte(xl>>16)]) ^ c.s2[byte(xl>>8)]) + c.s3[byte(xl)] ^ c.p[13] + xl ^= ((c.s0[byte(xr>>24)] + c.s1[byte(xr>>16)]) ^ c.s2[byte(xr>>8)]) + c.s3[byte(xr)] ^ c.p[14] + xr ^= ((c.s0[byte(xl>>24)] + c.s1[byte(xl>>16)]) ^ c.s2[byte(xl>>8)]) + c.s3[byte(xl)] ^ c.p[15] + xl ^= ((c.s0[byte(xr>>24)] + c.s1[byte(xr>>16)]) ^ c.s2[byte(xr>>8)]) + c.s3[byte(xr)] ^ c.p[16] + xr ^= c.p[17] + return xr, xl +} + +func decryptBlock(l, r uint32, c *Cipher) (uint32, uint32) { + xl, xr := l, r + xl ^= c.p[17] + xr ^= ((c.s0[byte(xl>>24)] + c.s1[byte(xl>>16)]) ^ c.s2[byte(xl>>8)]) + c.s3[byte(xl)] ^ c.p[16] + xl ^= ((c.s0[byte(xr>>24)] + c.s1[byte(xr>>16)]) ^ c.s2[byte(xr>>8)]) + c.s3[byte(xr)] ^ c.p[15] + xr ^= ((c.s0[byte(xl>>24)] + c.s1[byte(xl>>16)]) ^ c.s2[byte(xl>>8)]) + c.s3[byte(xl)] ^ c.p[14] + xl ^= ((c.s0[byte(xr>>24)] + c.s1[byte(xr>>16)]) ^ c.s2[byte(xr>>8)]) + c.s3[byte(xr)] ^ c.p[13] + xr ^= ((c.s0[byte(xl>>24)] + c.s1[byte(xl>>16)]) ^ c.s2[byte(xl>>8)]) + c.s3[byte(xl)] ^ c.p[12] + xl ^= ((c.s0[byte(xr>>24)] + c.s1[byte(xr>>16)]) ^ c.s2[byte(xr>>8)]) + c.s3[byte(xr)] ^ c.p[11] + xr ^= ((c.s0[byte(xl>>24)] + c.s1[byte(xl>>16)]) ^ c.s2[byte(xl>>8)]) + c.s3[byte(xl)] ^ c.p[10] + xl ^= ((c.s0[byte(xr>>24)] + c.s1[byte(xr>>16)]) ^ c.s2[byte(xr>>8)]) + c.s3[byte(xr)] ^ c.p[9] + xr ^= ((c.s0[byte(xl>>24)] + c.s1[byte(xl>>16)]) ^ c.s2[byte(xl>>8)]) + c.s3[byte(xl)] ^ c.p[8] + xl ^= ((c.s0[byte(xr>>24)] + c.s1[byte(xr>>16)]) ^ c.s2[byte(xr>>8)]) + c.s3[byte(xr)] ^ c.p[7] + xr ^= ((c.s0[byte(xl>>24)] + c.s1[byte(xl>>16)]) ^ c.s2[byte(xl>>8)]) + c.s3[byte(xl)] ^ c.p[6] + xl ^= ((c.s0[byte(xr>>24)] + c.s1[byte(xr>>16)]) ^ c.s2[byte(xr>>8)]) + c.s3[byte(xr)] ^ c.p[5] + xr ^= ((c.s0[byte(xl>>24)] + c.s1[byte(xl>>16)]) ^ c.s2[byte(xl>>8)]) + c.s3[byte(xl)] ^ c.p[4] + xl ^= ((c.s0[byte(xr>>24)] + c.s1[byte(xr>>16)]) ^ c.s2[byte(xr>>8)]) + c.s3[byte(xr)] ^ c.p[3] + xr ^= ((c.s0[byte(xl>>24)] + c.s1[byte(xl>>16)]) ^ c.s2[byte(xl>>8)]) + c.s3[byte(xl)] ^ c.p[2] + xl ^= ((c.s0[byte(xr>>24)] + c.s1[byte(xr>>16)]) ^ c.s2[byte(xr>>8)]) + c.s3[byte(xr)] ^ c.p[1] + xr ^= c.p[0] + return xr, xl +} + +func zero(x []uint32) { + for i := range x { + x[i] = 0 + } +} diff --git a/libgo/go/crypto/blowfish/blowfish_test.go b/libgo/go/crypto/blowfish/blowfish_test.go new file mode 100644 index 000000000..3a7ab6c2a --- /dev/null +++ b/libgo/go/crypto/blowfish/blowfish_test.go @@ -0,0 +1,192 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package blowfish + +import ( + "testing" +) + +type CryptTest struct { + key []byte + in []byte + out []byte +} + +// Test vector values are from http://www.schneier.com/code/vectors.txt. +var encryptTests = []CryptTest{ + { + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x4E, 0xF9, 0x97, 0x45, 0x61, 0x98, 0xDD, 0x78}}, + { + []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + []byte{0x51, 0x86, 0x6F, 0xD5, 0xB8, 0x5E, 0xCB, 0x8A}}, + { + []byte{0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + []byte{0x7D, 0x85, 0x6F, 0x9A, 0x61, 0x30, 0x63, 0xF2}}, + { + []byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}, + []byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}, + []byte{0x24, 0x66, 0xDD, 0x87, 0x8B, 0x96, 0x3C, 0x9D}}, + + { + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF}, + []byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}, + []byte{0x61, 0xF9, 0xC3, 0x80, 0x22, 0x81, 0xB0, 0x96}}, + { + []byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}, + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF}, + []byte{0x7D, 0x0C, 0xC6, 0x30, 0xAF, 0xDA, 0x1E, 0xC7}}, + { + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x4E, 0xF9, 0x97, 0x45, 0x61, 0x98, 0xDD, 0x78}}, + { + []byte{0xFE, 0xDC, 0xBA, 0x98, 0x76, 0x54, 0x32, 0x10}, + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF}, + []byte{0x0A, 0xCE, 0xAB, 0x0F, 0xC6, 0xA0, 0xA2, 0x8D}}, + { + []byte{0x7C, 0xA1, 0x10, 0x45, 0x4A, 0x1A, 0x6E, 0x57}, + []byte{0x01, 0xA1, 0xD6, 0xD0, 0x39, 0x77, 0x67, 0x42}, + []byte{0x59, 0xC6, 0x82, 0x45, 0xEB, 0x05, 0x28, 0x2B}}, + { + []byte{0x01, 0x31, 0xD9, 0x61, 0x9D, 0xC1, 0x37, 0x6E}, + []byte{0x5C, 0xD5, 0x4C, 0xA8, 0x3D, 0xEF, 0x57, 0xDA}, + []byte{0xB1, 0xB8, 0xCC, 0x0B, 0x25, 0x0F, 0x09, 0xA0}}, + { + []byte{0x07, 0xA1, 0x13, 0x3E, 0x4A, 0x0B, 0x26, 0x86}, + []byte{0x02, 0x48, 0xD4, 0x38, 0x06, 0xF6, 0x71, 0x72}, + []byte{0x17, 0x30, 0xE5, 0x77, 0x8B, 0xEA, 0x1D, 0xA4}}, + { + []byte{0x38, 0x49, 0x67, 0x4C, 0x26, 0x02, 0x31, 0x9E}, + []byte{0x51, 0x45, 0x4B, 0x58, 0x2D, 0xDF, 0x44, 0x0A}, + []byte{0xA2, 0x5E, 0x78, 0x56, 0xCF, 0x26, 0x51, 0xEB}}, + { + []byte{0x04, 0xB9, 0x15, 0xBA, 0x43, 0xFE, 0xB5, 0xB6}, + []byte{0x42, 0xFD, 0x44, 0x30, 0x59, 0x57, 0x7F, 0xA2}, + []byte{0x35, 0x38, 0x82, 0xB1, 0x09, 0xCE, 0x8F, 0x1A}}, + { + []byte{0x01, 0x13, 0xB9, 0x70, 0xFD, 0x34, 0xF2, 0xCE}, + []byte{0x05, 0x9B, 0x5E, 0x08, 0x51, 0xCF, 0x14, 0x3A}, + []byte{0x48, 0xF4, 0xD0, 0x88, 0x4C, 0x37, 0x99, 0x18}}, + { + []byte{0x01, 0x70, 0xF1, 0x75, 0x46, 0x8F, 0xB5, 0xE6}, + []byte{0x07, 0x56, 0xD8, 0xE0, 0x77, 0x47, 0x61, 0xD2}, + []byte{0x43, 0x21, 0x93, 0xB7, 0x89, 0x51, 0xFC, 0x98}}, + { + []byte{0x43, 0x29, 0x7F, 0xAD, 0x38, 0xE3, 0x73, 0xFE}, + []byte{0x76, 0x25, 0x14, 0xB8, 0x29, 0xBF, 0x48, 0x6A}, + []byte{0x13, 0xF0, 0x41, 0x54, 0xD6, 0x9D, 0x1A, 0xE5}}, + { + []byte{0x07, 0xA7, 0x13, 0x70, 0x45, 0xDA, 0x2A, 0x16}, + []byte{0x3B, 0xDD, 0x11, 0x90, 0x49, 0x37, 0x28, 0x02}, + []byte{0x2E, 0xED, 0xDA, 0x93, 0xFF, 0xD3, 0x9C, 0x79}}, + { + []byte{0x04, 0x68, 0x91, 0x04, 0xC2, 0xFD, 0x3B, 0x2F}, + []byte{0x26, 0x95, 0x5F, 0x68, 0x35, 0xAF, 0x60, 0x9A}, + []byte{0xD8, 0x87, 0xE0, 0x39, 0x3C, 0x2D, 0xA6, 0xE3}}, + { + []byte{0x37, 0xD0, 0x6B, 0xB5, 0x16, 0xCB, 0x75, 0x46}, + []byte{0x16, 0x4D, 0x5E, 0x40, 0x4F, 0x27, 0x52, 0x32}, + []byte{0x5F, 0x99, 0xD0, 0x4F, 0x5B, 0x16, 0x39, 0x69}}, + { + []byte{0x1F, 0x08, 0x26, 0x0D, 0x1A, 0xC2, 0x46, 0x5E}, + []byte{0x6B, 0x05, 0x6E, 0x18, 0x75, 0x9F, 0x5C, 0xCA}, + []byte{0x4A, 0x05, 0x7A, 0x3B, 0x24, 0xD3, 0x97, 0x7B}}, + { + []byte{0x58, 0x40, 0x23, 0x64, 0x1A, 0xBA, 0x61, 0x76}, + []byte{0x00, 0x4B, 0xD6, 0xEF, 0x09, 0x17, 0x60, 0x62}, + []byte{0x45, 0x20, 0x31, 0xC1, 0xE4, 0xFA, 0xDA, 0x8E}}, + { + []byte{0x02, 0x58, 0x16, 0x16, 0x46, 0x29, 0xB0, 0x07}, + []byte{0x48, 0x0D, 0x39, 0x00, 0x6E, 0xE7, 0x62, 0xF2}, + []byte{0x75, 0x55, 0xAE, 0x39, 0xF5, 0x9B, 0x87, 0xBD}}, + { + []byte{0x49, 0x79, 0x3E, 0xBC, 0x79, 0xB3, 0x25, 0x8F}, + []byte{0x43, 0x75, 0x40, 0xC8, 0x69, 0x8F, 0x3C, 0xFA}, + []byte{0x53, 0xC5, 0x5F, 0x9C, 0xB4, 0x9F, 0xC0, 0x19}}, + { + []byte{0x4F, 0xB0, 0x5E, 0x15, 0x15, 0xAB, 0x73, 0xA7}, + []byte{0x07, 0x2D, 0x43, 0xA0, 0x77, 0x07, 0x52, 0x92}, + []byte{0x7A, 0x8E, 0x7B, 0xFA, 0x93, 0x7E, 0x89, 0xA3}}, + { + []byte{0x49, 0xE9, 0x5D, 0x6D, 0x4C, 0xA2, 0x29, 0xBF}, + []byte{0x02, 0xFE, 0x55, 0x77, 0x81, 0x17, 0xF1, 0x2A}, + []byte{0xCF, 0x9C, 0x5D, 0x7A, 0x49, 0x86, 0xAD, 0xB5}}, + { + []byte{0x01, 0x83, 0x10, 0xDC, 0x40, 0x9B, 0x26, 0xD6}, + []byte{0x1D, 0x9D, 0x5C, 0x50, 0x18, 0xF7, 0x28, 0xC2}, + []byte{0xD1, 0xAB, 0xB2, 0x90, 0x65, 0x8B, 0xC7, 0x78}}, + { + []byte{0x1C, 0x58, 0x7F, 0x1C, 0x13, 0x92, 0x4F, 0xEF}, + []byte{0x30, 0x55, 0x32, 0x28, 0x6D, 0x6F, 0x29, 0x5A}, + []byte{0x55, 0xCB, 0x37, 0x74, 0xD1, 0x3E, 0xF2, 0x01}}, + { + []byte{0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF}, + []byte{0xFA, 0x34, 0xEC, 0x48, 0x47, 0xB2, 0x68, 0xB2}}, + { + []byte{0x1F, 0x1F, 0x1F, 0x1F, 0x0E, 0x0E, 0x0E, 0x0E}, + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF}, + []byte{0xA7, 0x90, 0x79, 0x51, 0x08, 0xEA, 0x3C, 0xAE}}, + { + []byte{0xE0, 0xFE, 0xE0, 0xFE, 0xF1, 0xFE, 0xF1, 0xFE}, + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF}, + []byte{0xC3, 0x9E, 0x07, 0x2D, 0x9F, 0xAC, 0x63, 0x1D}}, + { + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + []byte{0x01, 0x49, 0x33, 0xE0, 0xCD, 0xAF, 0xF6, 0xE4}}, + { + []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xF2, 0x1E, 0x9A, 0x77, 0xB7, 0x1C, 0x49, 0xBC}}, + { + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF}, + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x24, 0x59, 0x46, 0x88, 0x57, 0x54, 0x36, 0x9A}}, + { + []byte{0xFE, 0xDC, 0xBA, 0x98, 0x76, 0x54, 0x32, 0x10}, + []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + []byte{0x6B, 0x5C, 0x5A, 0x9C, 0x5D, 0x9E, 0x0A, 0x5A}}, +} + +func TestCipherEncrypt(t *testing.T) { + for i, tt := range encryptTests { + c, err := NewCipher(tt.key) + if err != nil { + t.Errorf("NewCipher(%d bytes) = %s", len(tt.key), err) + continue + } + ct := make([]byte, len(tt.out)) + c.Encrypt(ct, tt.in) + for j, v := range ct { + if v != tt.out[j] { + t.Errorf("Cipher.Encrypt, test vector #%d: cipher-text[%d] = %#x, expected %#x", i, j, v, tt.out[j]) + break + } + } + } +} + +func TestCipherDecrypt(t *testing.T) { + for i, tt := range encryptTests { + c, err := NewCipher(tt.key) + if err != nil { + t.Errorf("NewCipher(%d bytes) = %s", len(tt.key), err) + continue + } + pt := make([]byte, len(tt.in)) + c.Decrypt(pt, tt.out) + for j, v := range pt { + if v != tt.in[j] { + t.Errorf("Cipher.Decrypt, test vector #%d: plain-text[%d] = %#x, expected %#x", i, j, v, tt.in[j]) + break + } + } + } +} diff --git a/libgo/go/crypto/blowfish/cipher.go b/libgo/go/crypto/blowfish/cipher.go new file mode 100644 index 000000000..947f762d8 --- /dev/null +++ b/libgo/go/crypto/blowfish/cipher.go @@ -0,0 +1,79 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This package implements Bruce Schneier's Blowfish encryption algorithm. +package blowfish + +// The code is a port of Bruce Schneier's C implementation. +// See http://www.schneier.com/blowfish.html. + +import ( + "os" + "strconv" +) + +// The Blowfish block size in bytes. +const BlockSize = 8 + +// A Cipher is an instance of Blowfish encryption using a particular key. +type Cipher struct { + p [18]uint32 + s0, s1, s2, s3 [256]uint32 +} + +type KeySizeError int + +func (k KeySizeError) String() string { + return "crypto/blowfish: invalid key size " + strconv.Itoa(int(k)) +} + +// NewCipher creates and returns a Cipher. +// The key argument should be the Blowfish key, 4 to 56 bytes. +func NewCipher(key []byte) (*Cipher, os.Error) { + k := len(key) + if k < 4 || k > 56 { + return nil, KeySizeError(k) + } + var result Cipher + expandKey(key, &result) + return &result, nil +} + +// BlockSize returns the Blowfish block size, 8 bytes. +// It is necessary to satisfy the Cipher interface in the +// package "crypto/block". +func (c *Cipher) BlockSize() int { return BlockSize } + +// Encrypt encrypts the 8-byte buffer src using the key k +// and stores the result in dst. +// Note that for amounts of data larger than a block, +// it is not safe to just call Encrypt on successive blocks; +// instead, use an encryption mode like CBC (see crypto/block/cbc.go). +func (c *Cipher) Encrypt(dst, src []byte) { + l := uint32(src[0])<<24 | uint32(src[1])<<16 | uint32(src[2])<<8 | uint32(src[3]) + r := uint32(src[4])<<24 | uint32(src[5])<<16 | uint32(src[6])<<8 | uint32(src[7]) + l, r = encryptBlock(l, r, c) + dst[0], dst[1], dst[2], dst[3] = byte(l>>24), byte(l>>16), byte(l>>8), byte(l) + dst[4], dst[5], dst[6], dst[7] = byte(r>>24), byte(r>>16), byte(r>>8), byte(r) +} + +// Decrypt decrypts the 8-byte buffer src using the key k +// and stores the result in dst. +func (c *Cipher) Decrypt(dst, src []byte) { + l := uint32(src[0])<<24 | uint32(src[1])<<16 | uint32(src[2])<<8 | uint32(src[3]) + r := uint32(src[4])<<24 | uint32(src[5])<<16 | uint32(src[6])<<8 | uint32(src[7]) + l, r = decryptBlock(l, r, c) + dst[0], dst[1], dst[2], dst[3] = byte(l>>24), byte(l>>16), byte(l>>8), byte(l) + dst[4], dst[5], dst[6], dst[7] = byte(r>>24), byte(r>>16), byte(r>>8), byte(r) +} + +// Reset zeros the key data, so that it will no longer +// appear in the process's memory. +func (c *Cipher) Reset() { + zero(c.p[0:]) + zero(c.s0[0:]) + zero(c.s1[0:]) + zero(c.s2[0:]) + zero(c.s3[0:]) +} diff --git a/libgo/go/crypto/blowfish/const.go b/libgo/go/crypto/blowfish/const.go new file mode 100644 index 000000000..8c5ee4cb0 --- /dev/null +++ b/libgo/go/crypto/blowfish/const.go @@ -0,0 +1,199 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The startup permutation array and substitution boxes. +// They are the hexadecimal digits of PI; see: +// http://www.schneier.com/code/constants.txt. + +package blowfish + +var s0 = [256]uint32{ + 0xd1310ba6, 0x98dfb5ac, 0x2ffd72db, 0xd01adfb7, 0xb8e1afed, 0x6a267e96, + 0xba7c9045, 0xf12c7f99, 0x24a19947, 0xb3916cf7, 0x0801f2e2, 0x858efc16, + 0x636920d8, 0x71574e69, 0xa458fea3, 0xf4933d7e, 0x0d95748f, 0x728eb658, + 0x718bcd58, 0x82154aee, 0x7b54a41d, 0xc25a59b5, 0x9c30d539, 0x2af26013, + 0xc5d1b023, 0x286085f0, 0xca417918, 0xb8db38ef, 0x8e79dcb0, 0x603a180e, + 0x6c9e0e8b, 0xb01e8a3e, 0xd71577c1, 0xbd314b27, 0x78af2fda, 0x55605c60, + 0xe65525f3, 0xaa55ab94, 0x57489862, 0x63e81440, 0x55ca396a, 0x2aab10b6, + 0xb4cc5c34, 0x1141e8ce, 0xa15486af, 0x7c72e993, 0xb3ee1411, 0x636fbc2a, + 0x2ba9c55d, 0x741831f6, 0xce5c3e16, 0x9b87931e, 0xafd6ba33, 0x6c24cf5c, + 0x7a325381, 0x28958677, 0x3b8f4898, 0x6b4bb9af, 0xc4bfe81b, 0x66282193, + 0x61d809cc, 0xfb21a991, 0x487cac60, 0x5dec8032, 0xef845d5d, 0xe98575b1, + 0xdc262302, 0xeb651b88, 0x23893e81, 0xd396acc5, 0x0f6d6ff3, 0x83f44239, + 0x2e0b4482, 0xa4842004, 0x69c8f04a, 0x9e1f9b5e, 0x21c66842, 0xf6e96c9a, + 0x670c9c61, 0xabd388f0, 0x6a51a0d2, 0xd8542f68, 0x960fa728, 0xab5133a3, + 0x6eef0b6c, 0x137a3be4, 0xba3bf050, 0x7efb2a98, 0xa1f1651d, 0x39af0176, + 0x66ca593e, 0x82430e88, 0x8cee8619, 0x456f9fb4, 0x7d84a5c3, 0x3b8b5ebe, + 0xe06f75d8, 0x85c12073, 0x401a449f, 0x56c16aa6, 0x4ed3aa62, 0x363f7706, + 0x1bfedf72, 0x429b023d, 0x37d0d724, 0xd00a1248, 0xdb0fead3, 0x49f1c09b, + 0x075372c9, 0x80991b7b, 0x25d479d8, 0xf6e8def7, 0xe3fe501a, 0xb6794c3b, + 0x976ce0bd, 0x04c006ba, 0xc1a94fb6, 0x409f60c4, 0x5e5c9ec2, 0x196a2463, + 0x68fb6faf, 0x3e6c53b5, 0x1339b2eb, 0x3b52ec6f, 0x6dfc511f, 0x9b30952c, + 0xcc814544, 0xaf5ebd09, 0xbee3d004, 0xde334afd, 0x660f2807, 0x192e4bb3, + 0xc0cba857, 0x45c8740f, 0xd20b5f39, 0xb9d3fbdb, 0x5579c0bd, 0x1a60320a, + 0xd6a100c6, 0x402c7279, 0x679f25fe, 0xfb1fa3cc, 0x8ea5e9f8, 0xdb3222f8, + 0x3c7516df, 0xfd616b15, 0x2f501ec8, 0xad0552ab, 0x323db5fa, 0xfd238760, + 0x53317b48, 0x3e00df82, 0x9e5c57bb, 0xca6f8ca0, 0x1a87562e, 0xdf1769db, + 0xd542a8f6, 0x287effc3, 0xac6732c6, 0x8c4f5573, 0x695b27b0, 0xbbca58c8, + 0xe1ffa35d, 0xb8f011a0, 0x10fa3d98, 0xfd2183b8, 0x4afcb56c, 0x2dd1d35b, + 0x9a53e479, 0xb6f84565, 0xd28e49bc, 0x4bfb9790, 0xe1ddf2da, 0xa4cb7e33, + 0x62fb1341, 0xcee4c6e8, 0xef20cada, 0x36774c01, 0xd07e9efe, 0x2bf11fb4, + 0x95dbda4d, 0xae909198, 0xeaad8e71, 0x6b93d5a0, 0xd08ed1d0, 0xafc725e0, + 0x8e3c5b2f, 0x8e7594b7, 0x8ff6e2fb, 0xf2122b64, 0x8888b812, 0x900df01c, + 0x4fad5ea0, 0x688fc31c, 0xd1cff191, 0xb3a8c1ad, 0x2f2f2218, 0xbe0e1777, + 0xea752dfe, 0x8b021fa1, 0xe5a0cc0f, 0xb56f74e8, 0x18acf3d6, 0xce89e299, + 0xb4a84fe0, 0xfd13e0b7, 0x7cc43b81, 0xd2ada8d9, 0x165fa266, 0x80957705, + 0x93cc7314, 0x211a1477, 0xe6ad2065, 0x77b5fa86, 0xc75442f5, 0xfb9d35cf, + 0xebcdaf0c, 0x7b3e89a0, 0xd6411bd3, 0xae1e7e49, 0x00250e2d, 0x2071b35e, + 0x226800bb, 0x57b8e0af, 0x2464369b, 0xf009b91e, 0x5563911d, 0x59dfa6aa, + 0x78c14389, 0xd95a537f, 0x207d5ba2, 0x02e5b9c5, 0x83260376, 0x6295cfa9, + 0x11c81968, 0x4e734a41, 0xb3472dca, 0x7b14a94a, 0x1b510052, 0x9a532915, + 0xd60f573f, 0xbc9bc6e4, 0x2b60a476, 0x81e67400, 0x08ba6fb5, 0x571be91f, + 0xf296ec6b, 0x2a0dd915, 0xb6636521, 0xe7b9f9b6, 0xff34052e, 0xc5855664, + 0x53b02d5d, 0xa99f8fa1, 0x08ba4799, 0x6e85076a, +} + +var s1 = [256]uint32{ + 0x4b7a70e9, 0xb5b32944, 0xdb75092e, 0xc4192623, 0xad6ea6b0, 0x49a7df7d, + 0x9cee60b8, 0x8fedb266, 0xecaa8c71, 0x699a17ff, 0x5664526c, 0xc2b19ee1, + 0x193602a5, 0x75094c29, 0xa0591340, 0xe4183a3e, 0x3f54989a, 0x5b429d65, + 0x6b8fe4d6, 0x99f73fd6, 0xa1d29c07, 0xefe830f5, 0x4d2d38e6, 0xf0255dc1, + 0x4cdd2086, 0x8470eb26, 0x6382e9c6, 0x021ecc5e, 0x09686b3f, 0x3ebaefc9, + 0x3c971814, 0x6b6a70a1, 0x687f3584, 0x52a0e286, 0xb79c5305, 0xaa500737, + 0x3e07841c, 0x7fdeae5c, 0x8e7d44ec, 0x5716f2b8, 0xb03ada37, 0xf0500c0d, + 0xf01c1f04, 0x0200b3ff, 0xae0cf51a, 0x3cb574b2, 0x25837a58, 0xdc0921bd, + 0xd19113f9, 0x7ca92ff6, 0x94324773, 0x22f54701, 0x3ae5e581, 0x37c2dadc, + 0xc8b57634, 0x9af3dda7, 0xa9446146, 0x0fd0030e, 0xecc8c73e, 0xa4751e41, + 0xe238cd99, 0x3bea0e2f, 0x3280bba1, 0x183eb331, 0x4e548b38, 0x4f6db908, + 0x6f420d03, 0xf60a04bf, 0x2cb81290, 0x24977c79, 0x5679b072, 0xbcaf89af, + 0xde9a771f, 0xd9930810, 0xb38bae12, 0xdccf3f2e, 0x5512721f, 0x2e6b7124, + 0x501adde6, 0x9f84cd87, 0x7a584718, 0x7408da17, 0xbc9f9abc, 0xe94b7d8c, + 0xec7aec3a, 0xdb851dfa, 0x63094366, 0xc464c3d2, 0xef1c1847, 0x3215d908, + 0xdd433b37, 0x24c2ba16, 0x12a14d43, 0x2a65c451, 0x50940002, 0x133ae4dd, + 0x71dff89e, 0x10314e55, 0x81ac77d6, 0x5f11199b, 0x043556f1, 0xd7a3c76b, + 0x3c11183b, 0x5924a509, 0xf28fe6ed, 0x97f1fbfa, 0x9ebabf2c, 0x1e153c6e, + 0x86e34570, 0xeae96fb1, 0x860e5e0a, 0x5a3e2ab3, 0x771fe71c, 0x4e3d06fa, + 0x2965dcb9, 0x99e71d0f, 0x803e89d6, 0x5266c825, 0x2e4cc978, 0x9c10b36a, + 0xc6150eba, 0x94e2ea78, 0xa5fc3c53, 0x1e0a2df4, 0xf2f74ea7, 0x361d2b3d, + 0x1939260f, 0x19c27960, 0x5223a708, 0xf71312b6, 0xebadfe6e, 0xeac31f66, + 0xe3bc4595, 0xa67bc883, 0xb17f37d1, 0x018cff28, 0xc332ddef, 0xbe6c5aa5, + 0x65582185, 0x68ab9802, 0xeecea50f, 0xdb2f953b, 0x2aef7dad, 0x5b6e2f84, + 0x1521b628, 0x29076170, 0xecdd4775, 0x619f1510, 0x13cca830, 0xeb61bd96, + 0x0334fe1e, 0xaa0363cf, 0xb5735c90, 0x4c70a239, 0xd59e9e0b, 0xcbaade14, + 0xeecc86bc, 0x60622ca7, 0x9cab5cab, 0xb2f3846e, 0x648b1eaf, 0x19bdf0ca, + 0xa02369b9, 0x655abb50, 0x40685a32, 0x3c2ab4b3, 0x319ee9d5, 0xc021b8f7, + 0x9b540b19, 0x875fa099, 0x95f7997e, 0x623d7da8, 0xf837889a, 0x97e32d77, + 0x11ed935f, 0x16681281, 0x0e358829, 0xc7e61fd6, 0x96dedfa1, 0x7858ba99, + 0x57f584a5, 0x1b227263, 0x9b83c3ff, 0x1ac24696, 0xcdb30aeb, 0x532e3054, + 0x8fd948e4, 0x6dbc3128, 0x58ebf2ef, 0x34c6ffea, 0xfe28ed61, 0xee7c3c73, + 0x5d4a14d9, 0xe864b7e3, 0x42105d14, 0x203e13e0, 0x45eee2b6, 0xa3aaabea, + 0xdb6c4f15, 0xfacb4fd0, 0xc742f442, 0xef6abbb5, 0x654f3b1d, 0x41cd2105, + 0xd81e799e, 0x86854dc7, 0xe44b476a, 0x3d816250, 0xcf62a1f2, 0x5b8d2646, + 0xfc8883a0, 0xc1c7b6a3, 0x7f1524c3, 0x69cb7492, 0x47848a0b, 0x5692b285, + 0x095bbf00, 0xad19489d, 0x1462b174, 0x23820e00, 0x58428d2a, 0x0c55f5ea, + 0x1dadf43e, 0x233f7061, 0x3372f092, 0x8d937e41, 0xd65fecf1, 0x6c223bdb, + 0x7cde3759, 0xcbee7460, 0x4085f2a7, 0xce77326e, 0xa6078084, 0x19f8509e, + 0xe8efd855, 0x61d99735, 0xa969a7aa, 0xc50c06c2, 0x5a04abfc, 0x800bcadc, + 0x9e447a2e, 0xc3453484, 0xfdd56705, 0x0e1e9ec9, 0xdb73dbd3, 0x105588cd, + 0x675fda79, 0xe3674340, 0xc5c43465, 0x713e38d8, 0x3d28f89e, 0xf16dff20, + 0x153e21e7, 0x8fb03d4a, 0xe6e39f2b, 0xdb83adf7, +} + +var s2 = [256]uint32{ + 0xe93d5a68, 0x948140f7, 0xf64c261c, 0x94692934, 0x411520f7, 0x7602d4f7, + 0xbcf46b2e, 0xd4a20068, 0xd4082471, 0x3320f46a, 0x43b7d4b7, 0x500061af, + 0x1e39f62e, 0x97244546, 0x14214f74, 0xbf8b8840, 0x4d95fc1d, 0x96b591af, + 0x70f4ddd3, 0x66a02f45, 0xbfbc09ec, 0x03bd9785, 0x7fac6dd0, 0x31cb8504, + 0x96eb27b3, 0x55fd3941, 0xda2547e6, 0xabca0a9a, 0x28507825, 0x530429f4, + 0x0a2c86da, 0xe9b66dfb, 0x68dc1462, 0xd7486900, 0x680ec0a4, 0x27a18dee, + 0x4f3ffea2, 0xe887ad8c, 0xb58ce006, 0x7af4d6b6, 0xaace1e7c, 0xd3375fec, + 0xce78a399, 0x406b2a42, 0x20fe9e35, 0xd9f385b9, 0xee39d7ab, 0x3b124e8b, + 0x1dc9faf7, 0x4b6d1856, 0x26a36631, 0xeae397b2, 0x3a6efa74, 0xdd5b4332, + 0x6841e7f7, 0xca7820fb, 0xfb0af54e, 0xd8feb397, 0x454056ac, 0xba489527, + 0x55533a3a, 0x20838d87, 0xfe6ba9b7, 0xd096954b, 0x55a867bc, 0xa1159a58, + 0xcca92963, 0x99e1db33, 0xa62a4a56, 0x3f3125f9, 0x5ef47e1c, 0x9029317c, + 0xfdf8e802, 0x04272f70, 0x80bb155c, 0x05282ce3, 0x95c11548, 0xe4c66d22, + 0x48c1133f, 0xc70f86dc, 0x07f9c9ee, 0x41041f0f, 0x404779a4, 0x5d886e17, + 0x325f51eb, 0xd59bc0d1, 0xf2bcc18f, 0x41113564, 0x257b7834, 0x602a9c60, + 0xdff8e8a3, 0x1f636c1b, 0x0e12b4c2, 0x02e1329e, 0xaf664fd1, 0xcad18115, + 0x6b2395e0, 0x333e92e1, 0x3b240b62, 0xeebeb922, 0x85b2a20e, 0xe6ba0d99, + 0xde720c8c, 0x2da2f728, 0xd0127845, 0x95b794fd, 0x647d0862, 0xe7ccf5f0, + 0x5449a36f, 0x877d48fa, 0xc39dfd27, 0xf33e8d1e, 0x0a476341, 0x992eff74, + 0x3a6f6eab, 0xf4f8fd37, 0xa812dc60, 0xa1ebddf8, 0x991be14c, 0xdb6e6b0d, + 0xc67b5510, 0x6d672c37, 0x2765d43b, 0xdcd0e804, 0xf1290dc7, 0xcc00ffa3, + 0xb5390f92, 0x690fed0b, 0x667b9ffb, 0xcedb7d9c, 0xa091cf0b, 0xd9155ea3, + 0xbb132f88, 0x515bad24, 0x7b9479bf, 0x763bd6eb, 0x37392eb3, 0xcc115979, + 0x8026e297, 0xf42e312d, 0x6842ada7, 0xc66a2b3b, 0x12754ccc, 0x782ef11c, + 0x6a124237, 0xb79251e7, 0x06a1bbe6, 0x4bfb6350, 0x1a6b1018, 0x11caedfa, + 0x3d25bdd8, 0xe2e1c3c9, 0x44421659, 0x0a121386, 0xd90cec6e, 0xd5abea2a, + 0x64af674e, 0xda86a85f, 0xbebfe988, 0x64e4c3fe, 0x9dbc8057, 0xf0f7c086, + 0x60787bf8, 0x6003604d, 0xd1fd8346, 0xf6381fb0, 0x7745ae04, 0xd736fccc, + 0x83426b33, 0xf01eab71, 0xb0804187, 0x3c005e5f, 0x77a057be, 0xbde8ae24, + 0x55464299, 0xbf582e61, 0x4e58f48f, 0xf2ddfda2, 0xf474ef38, 0x8789bdc2, + 0x5366f9c3, 0xc8b38e74, 0xb475f255, 0x46fcd9b9, 0x7aeb2661, 0x8b1ddf84, + 0x846a0e79, 0x915f95e2, 0x466e598e, 0x20b45770, 0x8cd55591, 0xc902de4c, + 0xb90bace1, 0xbb8205d0, 0x11a86248, 0x7574a99e, 0xb77f19b6, 0xe0a9dc09, + 0x662d09a1, 0xc4324633, 0xe85a1f02, 0x09f0be8c, 0x4a99a025, 0x1d6efe10, + 0x1ab93d1d, 0x0ba5a4df, 0xa186f20f, 0x2868f169, 0xdcb7da83, 0x573906fe, + 0xa1e2ce9b, 0x4fcd7f52, 0x50115e01, 0xa70683fa, 0xa002b5c4, 0x0de6d027, + 0x9af88c27, 0x773f8641, 0xc3604c06, 0x61a806b5, 0xf0177a28, 0xc0f586e0, + 0x006058aa, 0x30dc7d62, 0x11e69ed7, 0x2338ea63, 0x53c2dd94, 0xc2c21634, + 0xbbcbee56, 0x90bcb6de, 0xebfc7da1, 0xce591d76, 0x6f05e409, 0x4b7c0188, + 0x39720a3d, 0x7c927c24, 0x86e3725f, 0x724d9db9, 0x1ac15bb4, 0xd39eb8fc, + 0xed545578, 0x08fca5b5, 0xd83d7cd3, 0x4dad0fc4, 0x1e50ef5e, 0xb161e6f8, + 0xa28514d9, 0x6c51133c, 0x6fd5c7e7, 0x56e14ec4, 0x362abfce, 0xddc6c837, + 0xd79a3234, 0x92638212, 0x670efa8e, 0x406000e0, +} + +var s3 = [256]uint32{ + 0x3a39ce37, 0xd3faf5cf, 0xabc27737, 0x5ac52d1b, 0x5cb0679e, 0x4fa33742, + 0xd3822740, 0x99bc9bbe, 0xd5118e9d, 0xbf0f7315, 0xd62d1c7e, 0xc700c47b, + 0xb78c1b6b, 0x21a19045, 0xb26eb1be, 0x6a366eb4, 0x5748ab2f, 0xbc946e79, + 0xc6a376d2, 0x6549c2c8, 0x530ff8ee, 0x468dde7d, 0xd5730a1d, 0x4cd04dc6, + 0x2939bbdb, 0xa9ba4650, 0xac9526e8, 0xbe5ee304, 0xa1fad5f0, 0x6a2d519a, + 0x63ef8ce2, 0x9a86ee22, 0xc089c2b8, 0x43242ef6, 0xa51e03aa, 0x9cf2d0a4, + 0x83c061ba, 0x9be96a4d, 0x8fe51550, 0xba645bd6, 0x2826a2f9, 0xa73a3ae1, + 0x4ba99586, 0xef5562e9, 0xc72fefd3, 0xf752f7da, 0x3f046f69, 0x77fa0a59, + 0x80e4a915, 0x87b08601, 0x9b09e6ad, 0x3b3ee593, 0xe990fd5a, 0x9e34d797, + 0x2cf0b7d9, 0x022b8b51, 0x96d5ac3a, 0x017da67d, 0xd1cf3ed6, 0x7c7d2d28, + 0x1f9f25cf, 0xadf2b89b, 0x5ad6b472, 0x5a88f54c, 0xe029ac71, 0xe019a5e6, + 0x47b0acfd, 0xed93fa9b, 0xe8d3c48d, 0x283b57cc, 0xf8d56629, 0x79132e28, + 0x785f0191, 0xed756055, 0xf7960e44, 0xe3d35e8c, 0x15056dd4, 0x88f46dba, + 0x03a16125, 0x0564f0bd, 0xc3eb9e15, 0x3c9057a2, 0x97271aec, 0xa93a072a, + 0x1b3f6d9b, 0x1e6321f5, 0xf59c66fb, 0x26dcf319, 0x7533d928, 0xb155fdf5, + 0x03563482, 0x8aba3cbb, 0x28517711, 0xc20ad9f8, 0xabcc5167, 0xccad925f, + 0x4de81751, 0x3830dc8e, 0x379d5862, 0x9320f991, 0xea7a90c2, 0xfb3e7bce, + 0x5121ce64, 0x774fbe32, 0xa8b6e37e, 0xc3293d46, 0x48de5369, 0x6413e680, + 0xa2ae0810, 0xdd6db224, 0x69852dfd, 0x09072166, 0xb39a460a, 0x6445c0dd, + 0x586cdecf, 0x1c20c8ae, 0x5bbef7dd, 0x1b588d40, 0xccd2017f, 0x6bb4e3bb, + 0xdda26a7e, 0x3a59ff45, 0x3e350a44, 0xbcb4cdd5, 0x72eacea8, 0xfa6484bb, + 0x8d6612ae, 0xbf3c6f47, 0xd29be463, 0x542f5d9e, 0xaec2771b, 0xf64e6370, + 0x740e0d8d, 0xe75b1357, 0xf8721671, 0xaf537d5d, 0x4040cb08, 0x4eb4e2cc, + 0x34d2466a, 0x0115af84, 0xe1b00428, 0x95983a1d, 0x06b89fb4, 0xce6ea048, + 0x6f3f3b82, 0x3520ab82, 0x011a1d4b, 0x277227f8, 0x611560b1, 0xe7933fdc, + 0xbb3a792b, 0x344525bd, 0xa08839e1, 0x51ce794b, 0x2f32c9b7, 0xa01fbac9, + 0xe01cc87e, 0xbcc7d1f6, 0xcf0111c3, 0xa1e8aac7, 0x1a908749, 0xd44fbd9a, + 0xd0dadecb, 0xd50ada38, 0x0339c32a, 0xc6913667, 0x8df9317c, 0xe0b12b4f, + 0xf79e59b7, 0x43f5bb3a, 0xf2d519ff, 0x27d9459c, 0xbf97222c, 0x15e6fc2a, + 0x0f91fc71, 0x9b941525, 0xfae59361, 0xceb69ceb, 0xc2a86459, 0x12baa8d1, + 0xb6c1075e, 0xe3056a0c, 0x10d25065, 0xcb03a442, 0xe0ec6e0e, 0x1698db3b, + 0x4c98a0be, 0x3278e964, 0x9f1f9532, 0xe0d392df, 0xd3a0342b, 0x8971f21e, + 0x1b0a7441, 0x4ba3348c, 0xc5be7120, 0xc37632d8, 0xdf359f8d, 0x9b992f2e, + 0xe60b6f47, 0x0fe3f11d, 0xe54cda54, 0x1edad891, 0xce6279cf, 0xcd3e7e6f, + 0x1618b166, 0xfd2c1d05, 0x848fd2c5, 0xf6fb2299, 0xf523f357, 0xa6327623, + 0x93a83531, 0x56cccd02, 0xacf08162, 0x5a75ebb5, 0x6e163697, 0x88d273cc, + 0xde966292, 0x81b949d0, 0x4c50901b, 0x71c65614, 0xe6c6c7bd, 0x327a140a, + 0x45e1d006, 0xc3f27b9a, 0xc9aa53fd, 0x62a80f00, 0xbb25bfe2, 0x35bdd2f6, + 0x71126905, 0xb2040222, 0xb6cbcf7c, 0xcd769c2b, 0x53113ec0, 0x1640e3d3, + 0x38abbd60, 0x2547adf0, 0xba38209c, 0xf746ce76, 0x77afa1c5, 0x20756060, + 0x85cbfe4e, 0x8ae88dd8, 0x7aaaf9b0, 0x4cf9aa7e, 0x1948c25c, 0x02fb8a8c, + 0x01c36ae4, 0xd6ebe1f9, 0x90d4f869, 0xa65cdea0, 0x3f09252d, 0xc208e69f, + 0xb74e6132, 0xce77e25b, 0x578fdfe3, 0x3ac372e6, +} + +var p = [18]uint32{ + 0x243f6a88, 0x85a308d3, 0x13198a2e, 0x03707344, 0xa4093822, 0x299f31d0, + 0x082efa98, 0xec4e6c89, 0x452821e6, 0x38d01377, 0xbe5466cf, 0x34e90c6c, + 0xc0ac29b7, 0xc97c50dd, 0x3f84d5b5, 0xb5470917, 0x9216d5d9, 0x8979fb1b, +} diff --git a/libgo/go/crypto/cast5/cast5.go b/libgo/go/crypto/cast5/cast5.go new file mode 100644 index 000000000..35f3e64b6 --- /dev/null +++ b/libgo/go/crypto/cast5/cast5.go @@ -0,0 +1,536 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This package implements CAST5, as defined in RFC 2144. CAST5 is a common +// OpenPGP cipher. +package cast5 + +import ( + "os" +) + +const BlockSize = 8 +const KeySize = 16 + +type Cipher struct { + masking [16]uint32 + rotate [16]uint8 +} + +func NewCipher(key []byte) (c *Cipher, err os.Error) { + if len(key) != KeySize { + return nil, os.ErrorString("CAST5: keys must be 16 bytes") + } + + c = new(Cipher) + c.keySchedule(key) + return +} + +func (c *Cipher) BlockSize() int { + return BlockSize +} + +// Reset zeros the key material in memory. +func (c *Cipher) Reset() { + for i := 0; i < 16; i++ { + c.masking[i] = 0 + c.rotate[i] = 0 + } +} + +func (c *Cipher) Encrypt(dst, src []byte) { + l := uint32(src[0])<<24 | uint32(src[1])<<16 | uint32(src[2])<<8 | uint32(src[3]) + r := uint32(src[4])<<24 | uint32(src[5])<<16 | uint32(src[6])<<8 | uint32(src[7]) + + l, r = r, l^f1(r, c.masking[0], c.rotate[0]) + l, r = r, l^f2(r, c.masking[1], c.rotate[1]) + l, r = r, l^f3(r, c.masking[2], c.rotate[2]) + l, r = r, l^f1(r, c.masking[3], c.rotate[3]) + + l, r = r, l^f2(r, c.masking[4], c.rotate[4]) + l, r = r, l^f3(r, c.masking[5], c.rotate[5]) + l, r = r, l^f1(r, c.masking[6], c.rotate[6]) + l, r = r, l^f2(r, c.masking[7], c.rotate[7]) + + l, r = r, l^f3(r, c.masking[8], c.rotate[8]) + l, r = r, l^f1(r, c.masking[9], c.rotate[9]) + l, r = r, l^f2(r, c.masking[10], c.rotate[10]) + l, r = r, l^f3(r, c.masking[11], c.rotate[11]) + + l, r = r, l^f1(r, c.masking[12], c.rotate[12]) + l, r = r, l^f2(r, c.masking[13], c.rotate[13]) + l, r = r, l^f3(r, c.masking[14], c.rotate[14]) + l, r = r, l^f1(r, c.masking[15], c.rotate[15]) + + dst[0] = uint8(r >> 24) + dst[1] = uint8(r >> 16) + dst[2] = uint8(r >> 8) + dst[3] = uint8(r) + dst[4] = uint8(l >> 24) + dst[5] = uint8(l >> 16) + dst[6] = uint8(l >> 8) + dst[7] = uint8(l) +} + +func (c *Cipher) Decrypt(dst, src []byte) { + l := uint32(src[0])<<24 | uint32(src[1])<<16 | uint32(src[2])<<8 | uint32(src[3]) + r := uint32(src[4])<<24 | uint32(src[5])<<16 | uint32(src[6])<<8 | uint32(src[7]) + + l, r = r, l^f1(r, c.masking[15], c.rotate[15]) + l, r = r, l^f3(r, c.masking[14], c.rotate[14]) + l, r = r, l^f2(r, c.masking[13], c.rotate[13]) + l, r = r, l^f1(r, c.masking[12], c.rotate[12]) + + l, r = r, l^f3(r, c.masking[11], c.rotate[11]) + l, r = r, l^f2(r, c.masking[10], c.rotate[10]) + l, r = r, l^f1(r, c.masking[9], c.rotate[9]) + l, r = r, l^f3(r, c.masking[8], c.rotate[8]) + + l, r = r, l^f2(r, c.masking[7], c.rotate[7]) + l, r = r, l^f1(r, c.masking[6], c.rotate[6]) + l, r = r, l^f3(r, c.masking[5], c.rotate[5]) + l, r = r, l^f2(r, c.masking[4], c.rotate[4]) + + l, r = r, l^f1(r, c.masking[3], c.rotate[3]) + l, r = r, l^f3(r, c.masking[2], c.rotate[2]) + l, r = r, l^f2(r, c.masking[1], c.rotate[1]) + l, r = r, l^f1(r, c.masking[0], c.rotate[0]) + + dst[0] = uint8(r >> 24) + dst[1] = uint8(r >> 16) + dst[2] = uint8(r >> 8) + dst[3] = uint8(r) + dst[4] = uint8(l >> 24) + dst[5] = uint8(l >> 16) + dst[6] = uint8(l >> 8) + dst[7] = uint8(l) +} + +type keyScheduleA [4][7]uint8 +type keyScheduleB [4][5]uint8 + +// keyScheduleRound contains the magic values for a round of the key schedule. +// The keyScheduleA deals with the lines like: +// z0z1z2z3 = x0x1x2x3 ^ S5[xD] ^ S6[xF] ^ S7[xC] ^ S8[xE] ^ S7[x8] +// Conceptually, both x and z are in the same array, x first. The first +// element describes which word of this array gets written to and the +// second, which word gets read. So, for the line above, it's "4, 0", because +// it's writing to the first word of z, which, being after x, is word 4, and +// reading from the first word of x: word 0. +// +// Next are the indexes into the S-boxes. Now the array is treated as bytes. So +// "xD" is 0xd. The first byte of z is written as "16 + 0", just to be clear +// that it's z that we're indexing. +// +// keyScheduleB deals with lines like: +// K1 = S5[z8] ^ S6[z9] ^ S7[z7] ^ S8[z6] ^ S5[z2] +// "K1" is ignored because key words are always written in order. So the five +// elements are the S-box indexes. They use the same form as in keyScheduleA, +// above. + +type keyScheduleRound struct{} +type keySchedule []keyScheduleRound + +var schedule = []struct { + a keyScheduleA + b keyScheduleB +}{ + { + keyScheduleA{ + {4, 0, 0xd, 0xf, 0xc, 0xe, 0x8}, + {5, 2, 16 + 0, 16 + 2, 16 + 1, 16 + 3, 0xa}, + {6, 3, 16 + 7, 16 + 6, 16 + 5, 16 + 4, 9}, + {7, 1, 16 + 0xa, 16 + 9, 16 + 0xb, 16 + 8, 0xb}, + }, + keyScheduleB{ + {16 + 8, 16 + 9, 16 + 7, 16 + 6, 16 + 2}, + {16 + 0xa, 16 + 0xb, 16 + 5, 16 + 4, 16 + 6}, + {16 + 0xc, 16 + 0xd, 16 + 3, 16 + 2, 16 + 9}, + {16 + 0xe, 16 + 0xf, 16 + 1, 16 + 0, 16 + 0xc}, + }, + }, + { + keyScheduleA{ + {0, 6, 16 + 5, 16 + 7, 16 + 4, 16 + 6, 16 + 0}, + {1, 4, 0, 2, 1, 3, 16 + 2}, + {2, 5, 7, 6, 5, 4, 16 + 1}, + {3, 7, 0xa, 9, 0xb, 8, 16 + 3}, + }, + keyScheduleB{ + {3, 2, 0xc, 0xd, 8}, + {1, 0, 0xe, 0xf, 0xd}, + {7, 6, 8, 9, 3}, + {5, 4, 0xa, 0xb, 7}, + }, + }, + { + keyScheduleA{ + {4, 0, 0xd, 0xf, 0xc, 0xe, 8}, + {5, 2, 16 + 0, 16 + 2, 16 + 1, 16 + 3, 0xa}, + {6, 3, 16 + 7, 16 + 6, 16 + 5, 16 + 4, 9}, + {7, 1, 16 + 0xa, 16 + 9, 16 + 0xb, 16 + 8, 0xb}, + }, + keyScheduleB{ + {16 + 3, 16 + 2, 16 + 0xc, 16 + 0xd, 16 + 9}, + {16 + 1, 16 + 0, 16 + 0xe, 16 + 0xf, 16 + 0xc}, + {16 + 7, 16 + 6, 16 + 8, 16 + 9, 16 + 2}, + {16 + 5, 16 + 4, 16 + 0xa, 16 + 0xb, 16 + 6}, + }, + }, + { + keyScheduleA{ + {0, 6, 16 + 5, 16 + 7, 16 + 4, 16 + 6, 16 + 0}, + {1, 4, 0, 2, 1, 3, 16 + 2}, + {2, 5, 7, 6, 5, 4, 16 + 1}, + {3, 7, 0xa, 9, 0xb, 8, 16 + 3}, + }, + keyScheduleB{ + {8, 9, 7, 6, 3}, + {0xa, 0xb, 5, 4, 7}, + {0xc, 0xd, 3, 2, 8}, + {0xe, 0xf, 1, 0, 0xd}, + }, + }, +} + +func (c *Cipher) keySchedule(in []byte) { + var t [8]uint32 + var k [32]uint32 + + for i := 0; i < 4; i++ { + j := i * 4 + t[i] = uint32(in[j])<<24 | uint32(in[j+1])<<16 | uint32(in[j+2])<<8 | uint32(in[j+3]) + } + + x := []byte{6, 7, 4, 5} + ki := 0 + + for half := 0; half < 2; half++ { + for _, round := range schedule { + for j := 0; j < 4; j++ { + var a [7]uint8 + copy(a[:], round.a[j][:]) + w := t[a[1]] + w ^= sBox[4][(t[a[2]>>2]>>(24-8*(a[2]&3)))&0xff] + w ^= sBox[5][(t[a[3]>>2]>>(24-8*(a[3]&3)))&0xff] + w ^= sBox[6][(t[a[4]>>2]>>(24-8*(a[4]&3)))&0xff] + w ^= sBox[7][(t[a[5]>>2]>>(24-8*(a[5]&3)))&0xff] + w ^= sBox[x[j]][(t[a[6]>>2]>>(24-8*(a[6]&3)))&0xff] + t[a[0]] = w + } + + for j := 0; j < 4; j++ { + var b [5]uint8 + copy(b[:], round.b[j][:]) + w := sBox[4][(t[b[0]>>2]>>(24-8*(b[0]&3)))&0xff] + w ^= sBox[5][(t[b[1]>>2]>>(24-8*(b[1]&3)))&0xff] + w ^= sBox[6][(t[b[2]>>2]>>(24-8*(b[2]&3)))&0xff] + w ^= sBox[7][(t[b[3]>>2]>>(24-8*(b[3]&3)))&0xff] + w ^= sBox[4+j][(t[b[4]>>2]>>(24-8*(b[4]&3)))&0xff] + k[ki] = w + ki++ + } + } + } + + for i := 0; i < 16; i++ { + c.masking[i] = k[i] + c.rotate[i] = uint8(k[16+i] & 0x1f) + } +} + +// These are the three 'f' functions. See RFC 2144, section 2.2. +func f1(d, m uint32, r uint8) uint32 { + t := m + d + I := (t << r) | (t >> (32 - r)) + return ((sBox[0][I>>24] ^ sBox[1][(I>>16)&0xff]) - sBox[2][(I>>8)&0xff]) + sBox[3][I&0xff] +} + +func f2(d, m uint32, r uint8) uint32 { + t := m ^ d + I := (t << r) | (t >> (32 - r)) + return ((sBox[0][I>>24] - sBox[1][(I>>16)&0xff]) + sBox[2][(I>>8)&0xff]) ^ sBox[3][I&0xff] +} + +func f3(d, m uint32, r uint8) uint32 { + t := m - d + I := (t << r) | (t >> (32 - r)) + return ((sBox[0][I>>24] + sBox[1][(I>>16)&0xff]) ^ sBox[2][(I>>8)&0xff]) - sBox[3][I&0xff] +} + +var sBox = [8][256]uint32{ + { + 0x30fb40d4, 0x9fa0ff0b, 0x6beccd2f, 0x3f258c7a, 0x1e213f2f, 0x9c004dd3, 0x6003e540, 0xcf9fc949, + 0xbfd4af27, 0x88bbbdb5, 0xe2034090, 0x98d09675, 0x6e63a0e0, 0x15c361d2, 0xc2e7661d, 0x22d4ff8e, + 0x28683b6f, 0xc07fd059, 0xff2379c8, 0x775f50e2, 0x43c340d3, 0xdf2f8656, 0x887ca41a, 0xa2d2bd2d, + 0xa1c9e0d6, 0x346c4819, 0x61b76d87, 0x22540f2f, 0x2abe32e1, 0xaa54166b, 0x22568e3a, 0xa2d341d0, + 0x66db40c8, 0xa784392f, 0x004dff2f, 0x2db9d2de, 0x97943fac, 0x4a97c1d8, 0x527644b7, 0xb5f437a7, + 0xb82cbaef, 0xd751d159, 0x6ff7f0ed, 0x5a097a1f, 0x827b68d0, 0x90ecf52e, 0x22b0c054, 0xbc8e5935, + 0x4b6d2f7f, 0x50bb64a2, 0xd2664910, 0xbee5812d, 0xb7332290, 0xe93b159f, 0xb48ee411, 0x4bff345d, + 0xfd45c240, 0xad31973f, 0xc4f6d02e, 0x55fc8165, 0xd5b1caad, 0xa1ac2dae, 0xa2d4b76d, 0xc19b0c50, + 0x882240f2, 0x0c6e4f38, 0xa4e4bfd7, 0x4f5ba272, 0x564c1d2f, 0xc59c5319, 0xb949e354, 0xb04669fe, + 0xb1b6ab8a, 0xc71358dd, 0x6385c545, 0x110f935d, 0x57538ad5, 0x6a390493, 0xe63d37e0, 0x2a54f6b3, + 0x3a787d5f, 0x6276a0b5, 0x19a6fcdf, 0x7a42206a, 0x29f9d4d5, 0xf61b1891, 0xbb72275e, 0xaa508167, + 0x38901091, 0xc6b505eb, 0x84c7cb8c, 0x2ad75a0f, 0x874a1427, 0xa2d1936b, 0x2ad286af, 0xaa56d291, + 0xd7894360, 0x425c750d, 0x93b39e26, 0x187184c9, 0x6c00b32d, 0x73e2bb14, 0xa0bebc3c, 0x54623779, + 0x64459eab, 0x3f328b82, 0x7718cf82, 0x59a2cea6, 0x04ee002e, 0x89fe78e6, 0x3fab0950, 0x325ff6c2, + 0x81383f05, 0x6963c5c8, 0x76cb5ad6, 0xd49974c9, 0xca180dcf, 0x380782d5, 0xc7fa5cf6, 0x8ac31511, + 0x35e79e13, 0x47da91d0, 0xf40f9086, 0xa7e2419e, 0x31366241, 0x051ef495, 0xaa573b04, 0x4a805d8d, + 0x548300d0, 0x00322a3c, 0xbf64cddf, 0xba57a68e, 0x75c6372b, 0x50afd341, 0xa7c13275, 0x915a0bf5, + 0x6b54bfab, 0x2b0b1426, 0xab4cc9d7, 0x449ccd82, 0xf7fbf265, 0xab85c5f3, 0x1b55db94, 0xaad4e324, + 0xcfa4bd3f, 0x2deaa3e2, 0x9e204d02, 0xc8bd25ac, 0xeadf55b3, 0xd5bd9e98, 0xe31231b2, 0x2ad5ad6c, + 0x954329de, 0xadbe4528, 0xd8710f69, 0xaa51c90f, 0xaa786bf6, 0x22513f1e, 0xaa51a79b, 0x2ad344cc, + 0x7b5a41f0, 0xd37cfbad, 0x1b069505, 0x41ece491, 0xb4c332e6, 0x032268d4, 0xc9600acc, 0xce387e6d, + 0xbf6bb16c, 0x6a70fb78, 0x0d03d9c9, 0xd4df39de, 0xe01063da, 0x4736f464, 0x5ad328d8, 0xb347cc96, + 0x75bb0fc3, 0x98511bfb, 0x4ffbcc35, 0xb58bcf6a, 0xe11f0abc, 0xbfc5fe4a, 0xa70aec10, 0xac39570a, + 0x3f04442f, 0x6188b153, 0xe0397a2e, 0x5727cb79, 0x9ceb418f, 0x1cacd68d, 0x2ad37c96, 0x0175cb9d, + 0xc69dff09, 0xc75b65f0, 0xd9db40d8, 0xec0e7779, 0x4744ead4, 0xb11c3274, 0xdd24cb9e, 0x7e1c54bd, + 0xf01144f9, 0xd2240eb1, 0x9675b3fd, 0xa3ac3755, 0xd47c27af, 0x51c85f4d, 0x56907596, 0xa5bb15e6, + 0x580304f0, 0xca042cf1, 0x011a37ea, 0x8dbfaadb, 0x35ba3e4a, 0x3526ffa0, 0xc37b4d09, 0xbc306ed9, + 0x98a52666, 0x5648f725, 0xff5e569d, 0x0ced63d0, 0x7c63b2cf, 0x700b45e1, 0xd5ea50f1, 0x85a92872, + 0xaf1fbda7, 0xd4234870, 0xa7870bf3, 0x2d3b4d79, 0x42e04198, 0x0cd0ede7, 0x26470db8, 0xf881814c, + 0x474d6ad7, 0x7c0c5e5c, 0xd1231959, 0x381b7298, 0xf5d2f4db, 0xab838653, 0x6e2f1e23, 0x83719c9e, + 0xbd91e046, 0x9a56456e, 0xdc39200c, 0x20c8c571, 0x962bda1c, 0xe1e696ff, 0xb141ab08, 0x7cca89b9, + 0x1a69e783, 0x02cc4843, 0xa2f7c579, 0x429ef47d, 0x427b169c, 0x5ac9f049, 0xdd8f0f00, 0x5c8165bf, + }, + { + 0x1f201094, 0xef0ba75b, 0x69e3cf7e, 0x393f4380, 0xfe61cf7a, 0xeec5207a, 0x55889c94, 0x72fc0651, + 0xada7ef79, 0x4e1d7235, 0xd55a63ce, 0xde0436ba, 0x99c430ef, 0x5f0c0794, 0x18dcdb7d, 0xa1d6eff3, + 0xa0b52f7b, 0x59e83605, 0xee15b094, 0xe9ffd909, 0xdc440086, 0xef944459, 0xba83ccb3, 0xe0c3cdfb, + 0xd1da4181, 0x3b092ab1, 0xf997f1c1, 0xa5e6cf7b, 0x01420ddb, 0xe4e7ef5b, 0x25a1ff41, 0xe180f806, + 0x1fc41080, 0x179bee7a, 0xd37ac6a9, 0xfe5830a4, 0x98de8b7f, 0x77e83f4e, 0x79929269, 0x24fa9f7b, + 0xe113c85b, 0xacc40083, 0xd7503525, 0xf7ea615f, 0x62143154, 0x0d554b63, 0x5d681121, 0xc866c359, + 0x3d63cf73, 0xcee234c0, 0xd4d87e87, 0x5c672b21, 0x071f6181, 0x39f7627f, 0x361e3084, 0xe4eb573b, + 0x602f64a4, 0xd63acd9c, 0x1bbc4635, 0x9e81032d, 0x2701f50c, 0x99847ab4, 0xa0e3df79, 0xba6cf38c, + 0x10843094, 0x2537a95e, 0xf46f6ffe, 0xa1ff3b1f, 0x208cfb6a, 0x8f458c74, 0xd9e0a227, 0x4ec73a34, + 0xfc884f69, 0x3e4de8df, 0xef0e0088, 0x3559648d, 0x8a45388c, 0x1d804366, 0x721d9bfd, 0xa58684bb, + 0xe8256333, 0x844e8212, 0x128d8098, 0xfed33fb4, 0xce280ae1, 0x27e19ba5, 0xd5a6c252, 0xe49754bd, + 0xc5d655dd, 0xeb667064, 0x77840b4d, 0xa1b6a801, 0x84db26a9, 0xe0b56714, 0x21f043b7, 0xe5d05860, + 0x54f03084, 0x066ff472, 0xa31aa153, 0xdadc4755, 0xb5625dbf, 0x68561be6, 0x83ca6b94, 0x2d6ed23b, + 0xeccf01db, 0xa6d3d0ba, 0xb6803d5c, 0xaf77a709, 0x33b4a34c, 0x397bc8d6, 0x5ee22b95, 0x5f0e5304, + 0x81ed6f61, 0x20e74364, 0xb45e1378, 0xde18639b, 0x881ca122, 0xb96726d1, 0x8049a7e8, 0x22b7da7b, + 0x5e552d25, 0x5272d237, 0x79d2951c, 0xc60d894c, 0x488cb402, 0x1ba4fe5b, 0xa4b09f6b, 0x1ca815cf, + 0xa20c3005, 0x8871df63, 0xb9de2fcb, 0x0cc6c9e9, 0x0beeff53, 0xe3214517, 0xb4542835, 0x9f63293c, + 0xee41e729, 0x6e1d2d7c, 0x50045286, 0x1e6685f3, 0xf33401c6, 0x30a22c95, 0x31a70850, 0x60930f13, + 0x73f98417, 0xa1269859, 0xec645c44, 0x52c877a9, 0xcdff33a6, 0xa02b1741, 0x7cbad9a2, 0x2180036f, + 0x50d99c08, 0xcb3f4861, 0xc26bd765, 0x64a3f6ab, 0x80342676, 0x25a75e7b, 0xe4e6d1fc, 0x20c710e6, + 0xcdf0b680, 0x17844d3b, 0x31eef84d, 0x7e0824e4, 0x2ccb49eb, 0x846a3bae, 0x8ff77888, 0xee5d60f6, + 0x7af75673, 0x2fdd5cdb, 0xa11631c1, 0x30f66f43, 0xb3faec54, 0x157fd7fa, 0xef8579cc, 0xd152de58, + 0xdb2ffd5e, 0x8f32ce19, 0x306af97a, 0x02f03ef8, 0x99319ad5, 0xc242fa0f, 0xa7e3ebb0, 0xc68e4906, + 0xb8da230c, 0x80823028, 0xdcdef3c8, 0xd35fb171, 0x088a1bc8, 0xbec0c560, 0x61a3c9e8, 0xbca8f54d, + 0xc72feffa, 0x22822e99, 0x82c570b4, 0xd8d94e89, 0x8b1c34bc, 0x301e16e6, 0x273be979, 0xb0ffeaa6, + 0x61d9b8c6, 0x00b24869, 0xb7ffce3f, 0x08dc283b, 0x43daf65a, 0xf7e19798, 0x7619b72f, 0x8f1c9ba4, + 0xdc8637a0, 0x16a7d3b1, 0x9fc393b7, 0xa7136eeb, 0xc6bcc63e, 0x1a513742, 0xef6828bc, 0x520365d6, + 0x2d6a77ab, 0x3527ed4b, 0x821fd216, 0x095c6e2e, 0xdb92f2fb, 0x5eea29cb, 0x145892f5, 0x91584f7f, + 0x5483697b, 0x2667a8cc, 0x85196048, 0x8c4bacea, 0x833860d4, 0x0d23e0f9, 0x6c387e8a, 0x0ae6d249, + 0xb284600c, 0xd835731d, 0xdcb1c647, 0xac4c56ea, 0x3ebd81b3, 0x230eabb0, 0x6438bc87, 0xf0b5b1fa, + 0x8f5ea2b3, 0xfc184642, 0x0a036b7a, 0x4fb089bd, 0x649da589, 0xa345415e, 0x5c038323, 0x3e5d3bb9, + 0x43d79572, 0x7e6dd07c, 0x06dfdf1e, 0x6c6cc4ef, 0x7160a539, 0x73bfbe70, 0x83877605, 0x4523ecf1, + }, + { + 0x8defc240, 0x25fa5d9f, 0xeb903dbf, 0xe810c907, 0x47607fff, 0x369fe44b, 0x8c1fc644, 0xaececa90, + 0xbeb1f9bf, 0xeefbcaea, 0xe8cf1950, 0x51df07ae, 0x920e8806, 0xf0ad0548, 0xe13c8d83, 0x927010d5, + 0x11107d9f, 0x07647db9, 0xb2e3e4d4, 0x3d4f285e, 0xb9afa820, 0xfade82e0, 0xa067268b, 0x8272792e, + 0x553fb2c0, 0x489ae22b, 0xd4ef9794, 0x125e3fbc, 0x21fffcee, 0x825b1bfd, 0x9255c5ed, 0x1257a240, + 0x4e1a8302, 0xbae07fff, 0x528246e7, 0x8e57140e, 0x3373f7bf, 0x8c9f8188, 0xa6fc4ee8, 0xc982b5a5, + 0xa8c01db7, 0x579fc264, 0x67094f31, 0xf2bd3f5f, 0x40fff7c1, 0x1fb78dfc, 0x8e6bd2c1, 0x437be59b, + 0x99b03dbf, 0xb5dbc64b, 0x638dc0e6, 0x55819d99, 0xa197c81c, 0x4a012d6e, 0xc5884a28, 0xccc36f71, + 0xb843c213, 0x6c0743f1, 0x8309893c, 0x0feddd5f, 0x2f7fe850, 0xd7c07f7e, 0x02507fbf, 0x5afb9a04, + 0xa747d2d0, 0x1651192e, 0xaf70bf3e, 0x58c31380, 0x5f98302e, 0x727cc3c4, 0x0a0fb402, 0x0f7fef82, + 0x8c96fdad, 0x5d2c2aae, 0x8ee99a49, 0x50da88b8, 0x8427f4a0, 0x1eac5790, 0x796fb449, 0x8252dc15, + 0xefbd7d9b, 0xa672597d, 0xada840d8, 0x45f54504, 0xfa5d7403, 0xe83ec305, 0x4f91751a, 0x925669c2, + 0x23efe941, 0xa903f12e, 0x60270df2, 0x0276e4b6, 0x94fd6574, 0x927985b2, 0x8276dbcb, 0x02778176, + 0xf8af918d, 0x4e48f79e, 0x8f616ddf, 0xe29d840e, 0x842f7d83, 0x340ce5c8, 0x96bbb682, 0x93b4b148, + 0xef303cab, 0x984faf28, 0x779faf9b, 0x92dc560d, 0x224d1e20, 0x8437aa88, 0x7d29dc96, 0x2756d3dc, + 0x8b907cee, 0xb51fd240, 0xe7c07ce3, 0xe566b4a1, 0xc3e9615e, 0x3cf8209d, 0x6094d1e3, 0xcd9ca341, + 0x5c76460e, 0x00ea983b, 0xd4d67881, 0xfd47572c, 0xf76cedd9, 0xbda8229c, 0x127dadaa, 0x438a074e, + 0x1f97c090, 0x081bdb8a, 0x93a07ebe, 0xb938ca15, 0x97b03cff, 0x3dc2c0f8, 0x8d1ab2ec, 0x64380e51, + 0x68cc7bfb, 0xd90f2788, 0x12490181, 0x5de5ffd4, 0xdd7ef86a, 0x76a2e214, 0xb9a40368, 0x925d958f, + 0x4b39fffa, 0xba39aee9, 0xa4ffd30b, 0xfaf7933b, 0x6d498623, 0x193cbcfa, 0x27627545, 0x825cf47a, + 0x61bd8ba0, 0xd11e42d1, 0xcead04f4, 0x127ea392, 0x10428db7, 0x8272a972, 0x9270c4a8, 0x127de50b, + 0x285ba1c8, 0x3c62f44f, 0x35c0eaa5, 0xe805d231, 0x428929fb, 0xb4fcdf82, 0x4fb66a53, 0x0e7dc15b, + 0x1f081fab, 0x108618ae, 0xfcfd086d, 0xf9ff2889, 0x694bcc11, 0x236a5cae, 0x12deca4d, 0x2c3f8cc5, + 0xd2d02dfe, 0xf8ef5896, 0xe4cf52da, 0x95155b67, 0x494a488c, 0xb9b6a80c, 0x5c8f82bc, 0x89d36b45, + 0x3a609437, 0xec00c9a9, 0x44715253, 0x0a874b49, 0xd773bc40, 0x7c34671c, 0x02717ef6, 0x4feb5536, + 0xa2d02fff, 0xd2bf60c4, 0xd43f03c0, 0x50b4ef6d, 0x07478cd1, 0x006e1888, 0xa2e53f55, 0xb9e6d4bc, + 0xa2048016, 0x97573833, 0xd7207d67, 0xde0f8f3d, 0x72f87b33, 0xabcc4f33, 0x7688c55d, 0x7b00a6b0, + 0x947b0001, 0x570075d2, 0xf9bb88f8, 0x8942019e, 0x4264a5ff, 0x856302e0, 0x72dbd92b, 0xee971b69, + 0x6ea22fde, 0x5f08ae2b, 0xaf7a616d, 0xe5c98767, 0xcf1febd2, 0x61efc8c2, 0xf1ac2571, 0xcc8239c2, + 0x67214cb8, 0xb1e583d1, 0xb7dc3e62, 0x7f10bdce, 0xf90a5c38, 0x0ff0443d, 0x606e6dc6, 0x60543a49, + 0x5727c148, 0x2be98a1d, 0x8ab41738, 0x20e1be24, 0xaf96da0f, 0x68458425, 0x99833be5, 0x600d457d, + 0x282f9350, 0x8334b362, 0xd91d1120, 0x2b6d8da0, 0x642b1e31, 0x9c305a00, 0x52bce688, 0x1b03588a, + 0xf7baefd5, 0x4142ed9c, 0xa4315c11, 0x83323ec5, 0xdfef4636, 0xa133c501, 0xe9d3531c, 0xee353783, + }, + { + 0x9db30420, 0x1fb6e9de, 0xa7be7bef, 0xd273a298, 0x4a4f7bdb, 0x64ad8c57, 0x85510443, 0xfa020ed1, + 0x7e287aff, 0xe60fb663, 0x095f35a1, 0x79ebf120, 0xfd059d43, 0x6497b7b1, 0xf3641f63, 0x241e4adf, + 0x28147f5f, 0x4fa2b8cd, 0xc9430040, 0x0cc32220, 0xfdd30b30, 0xc0a5374f, 0x1d2d00d9, 0x24147b15, + 0xee4d111a, 0x0fca5167, 0x71ff904c, 0x2d195ffe, 0x1a05645f, 0x0c13fefe, 0x081b08ca, 0x05170121, + 0x80530100, 0xe83e5efe, 0xac9af4f8, 0x7fe72701, 0xd2b8ee5f, 0x06df4261, 0xbb9e9b8a, 0x7293ea25, + 0xce84ffdf, 0xf5718801, 0x3dd64b04, 0xa26f263b, 0x7ed48400, 0x547eebe6, 0x446d4ca0, 0x6cf3d6f5, + 0x2649abdf, 0xaea0c7f5, 0x36338cc1, 0x503f7e93, 0xd3772061, 0x11b638e1, 0x72500e03, 0xf80eb2bb, + 0xabe0502e, 0xec8d77de, 0x57971e81, 0xe14f6746, 0xc9335400, 0x6920318f, 0x081dbb99, 0xffc304a5, + 0x4d351805, 0x7f3d5ce3, 0xa6c866c6, 0x5d5bcca9, 0xdaec6fea, 0x9f926f91, 0x9f46222f, 0x3991467d, + 0xa5bf6d8e, 0x1143c44f, 0x43958302, 0xd0214eeb, 0x022083b8, 0x3fb6180c, 0x18f8931e, 0x281658e6, + 0x26486e3e, 0x8bd78a70, 0x7477e4c1, 0xb506e07c, 0xf32d0a25, 0x79098b02, 0xe4eabb81, 0x28123b23, + 0x69dead38, 0x1574ca16, 0xdf871b62, 0x211c40b7, 0xa51a9ef9, 0x0014377b, 0x041e8ac8, 0x09114003, + 0xbd59e4d2, 0xe3d156d5, 0x4fe876d5, 0x2f91a340, 0x557be8de, 0x00eae4a7, 0x0ce5c2ec, 0x4db4bba6, + 0xe756bdff, 0xdd3369ac, 0xec17b035, 0x06572327, 0x99afc8b0, 0x56c8c391, 0x6b65811c, 0x5e146119, + 0x6e85cb75, 0xbe07c002, 0xc2325577, 0x893ff4ec, 0x5bbfc92d, 0xd0ec3b25, 0xb7801ab7, 0x8d6d3b24, + 0x20c763ef, 0xc366a5fc, 0x9c382880, 0x0ace3205, 0xaac9548a, 0xeca1d7c7, 0x041afa32, 0x1d16625a, + 0x6701902c, 0x9b757a54, 0x31d477f7, 0x9126b031, 0x36cc6fdb, 0xc70b8b46, 0xd9e66a48, 0x56e55a79, + 0x026a4ceb, 0x52437eff, 0x2f8f76b4, 0x0df980a5, 0x8674cde3, 0xedda04eb, 0x17a9be04, 0x2c18f4df, + 0xb7747f9d, 0xab2af7b4, 0xefc34d20, 0x2e096b7c, 0x1741a254, 0xe5b6a035, 0x213d42f6, 0x2c1c7c26, + 0x61c2f50f, 0x6552daf9, 0xd2c231f8, 0x25130f69, 0xd8167fa2, 0x0418f2c8, 0x001a96a6, 0x0d1526ab, + 0x63315c21, 0x5e0a72ec, 0x49bafefd, 0x187908d9, 0x8d0dbd86, 0x311170a7, 0x3e9b640c, 0xcc3e10d7, + 0xd5cad3b6, 0x0caec388, 0xf73001e1, 0x6c728aff, 0x71eae2a1, 0x1f9af36e, 0xcfcbd12f, 0xc1de8417, + 0xac07be6b, 0xcb44a1d8, 0x8b9b0f56, 0x013988c3, 0xb1c52fca, 0xb4be31cd, 0xd8782806, 0x12a3a4e2, + 0x6f7de532, 0x58fd7eb6, 0xd01ee900, 0x24adffc2, 0xf4990fc5, 0x9711aac5, 0x001d7b95, 0x82e5e7d2, + 0x109873f6, 0x00613096, 0xc32d9521, 0xada121ff, 0x29908415, 0x7fbb977f, 0xaf9eb3db, 0x29c9ed2a, + 0x5ce2a465, 0xa730f32c, 0xd0aa3fe8, 0x8a5cc091, 0xd49e2ce7, 0x0ce454a9, 0xd60acd86, 0x015f1919, + 0x77079103, 0xdea03af6, 0x78a8565e, 0xdee356df, 0x21f05cbe, 0x8b75e387, 0xb3c50651, 0xb8a5c3ef, + 0xd8eeb6d2, 0xe523be77, 0xc2154529, 0x2f69efdf, 0xafe67afb, 0xf470c4b2, 0xf3e0eb5b, 0xd6cc9876, + 0x39e4460c, 0x1fda8538, 0x1987832f, 0xca007367, 0xa99144f8, 0x296b299e, 0x492fc295, 0x9266beab, + 0xb5676e69, 0x9bd3ddda, 0xdf7e052f, 0xdb25701c, 0x1b5e51ee, 0xf65324e6, 0x6afce36c, 0x0316cc04, + 0x8644213e, 0xb7dc59d0, 0x7965291f, 0xccd6fd43, 0x41823979, 0x932bcdf6, 0xb657c34d, 0x4edfd282, + 0x7ae5290c, 0x3cb9536b, 0x851e20fe, 0x9833557e, 0x13ecf0b0, 0xd3ffb372, 0x3f85c5c1, 0x0aef7ed2, + }, + { + 0x7ec90c04, 0x2c6e74b9, 0x9b0e66df, 0xa6337911, 0xb86a7fff, 0x1dd358f5, 0x44dd9d44, 0x1731167f, + 0x08fbf1fa, 0xe7f511cc, 0xd2051b00, 0x735aba00, 0x2ab722d8, 0x386381cb, 0xacf6243a, 0x69befd7a, + 0xe6a2e77f, 0xf0c720cd, 0xc4494816, 0xccf5c180, 0x38851640, 0x15b0a848, 0xe68b18cb, 0x4caadeff, + 0x5f480a01, 0x0412b2aa, 0x259814fc, 0x41d0efe2, 0x4e40b48d, 0x248eb6fb, 0x8dba1cfe, 0x41a99b02, + 0x1a550a04, 0xba8f65cb, 0x7251f4e7, 0x95a51725, 0xc106ecd7, 0x97a5980a, 0xc539b9aa, 0x4d79fe6a, + 0xf2f3f763, 0x68af8040, 0xed0c9e56, 0x11b4958b, 0xe1eb5a88, 0x8709e6b0, 0xd7e07156, 0x4e29fea7, + 0x6366e52d, 0x02d1c000, 0xc4ac8e05, 0x9377f571, 0x0c05372a, 0x578535f2, 0x2261be02, 0xd642a0c9, + 0xdf13a280, 0x74b55bd2, 0x682199c0, 0xd421e5ec, 0x53fb3ce8, 0xc8adedb3, 0x28a87fc9, 0x3d959981, + 0x5c1ff900, 0xfe38d399, 0x0c4eff0b, 0x062407ea, 0xaa2f4fb1, 0x4fb96976, 0x90c79505, 0xb0a8a774, + 0xef55a1ff, 0xe59ca2c2, 0xa6b62d27, 0xe66a4263, 0xdf65001f, 0x0ec50966, 0xdfdd55bc, 0x29de0655, + 0x911e739a, 0x17af8975, 0x32c7911c, 0x89f89468, 0x0d01e980, 0x524755f4, 0x03b63cc9, 0x0cc844b2, + 0xbcf3f0aa, 0x87ac36e9, 0xe53a7426, 0x01b3d82b, 0x1a9e7449, 0x64ee2d7e, 0xcddbb1da, 0x01c94910, + 0xb868bf80, 0x0d26f3fd, 0x9342ede7, 0x04a5c284, 0x636737b6, 0x50f5b616, 0xf24766e3, 0x8eca36c1, + 0x136e05db, 0xfef18391, 0xfb887a37, 0xd6e7f7d4, 0xc7fb7dc9, 0x3063fcdf, 0xb6f589de, 0xec2941da, + 0x26e46695, 0xb7566419, 0xf654efc5, 0xd08d58b7, 0x48925401, 0xc1bacb7f, 0xe5ff550f, 0xb6083049, + 0x5bb5d0e8, 0x87d72e5a, 0xab6a6ee1, 0x223a66ce, 0xc62bf3cd, 0x9e0885f9, 0x68cb3e47, 0x086c010f, + 0xa21de820, 0xd18b69de, 0xf3f65777, 0xfa02c3f6, 0x407edac3, 0xcbb3d550, 0x1793084d, 0xb0d70eba, + 0x0ab378d5, 0xd951fb0c, 0xded7da56, 0x4124bbe4, 0x94ca0b56, 0x0f5755d1, 0xe0e1e56e, 0x6184b5be, + 0x580a249f, 0x94f74bc0, 0xe327888e, 0x9f7b5561, 0xc3dc0280, 0x05687715, 0x646c6bd7, 0x44904db3, + 0x66b4f0a3, 0xc0f1648a, 0x697ed5af, 0x49e92ff6, 0x309e374f, 0x2cb6356a, 0x85808573, 0x4991f840, + 0x76f0ae02, 0x083be84d, 0x28421c9a, 0x44489406, 0x736e4cb8, 0xc1092910, 0x8bc95fc6, 0x7d869cf4, + 0x134f616f, 0x2e77118d, 0xb31b2be1, 0xaa90b472, 0x3ca5d717, 0x7d161bba, 0x9cad9010, 0xaf462ba2, + 0x9fe459d2, 0x45d34559, 0xd9f2da13, 0xdbc65487, 0xf3e4f94e, 0x176d486f, 0x097c13ea, 0x631da5c7, + 0x445f7382, 0x175683f4, 0xcdc66a97, 0x70be0288, 0xb3cdcf72, 0x6e5dd2f3, 0x20936079, 0x459b80a5, + 0xbe60e2db, 0xa9c23101, 0xeba5315c, 0x224e42f2, 0x1c5c1572, 0xf6721b2c, 0x1ad2fff3, 0x8c25404e, + 0x324ed72f, 0x4067b7fd, 0x0523138e, 0x5ca3bc78, 0xdc0fd66e, 0x75922283, 0x784d6b17, 0x58ebb16e, + 0x44094f85, 0x3f481d87, 0xfcfeae7b, 0x77b5ff76, 0x8c2302bf, 0xaaf47556, 0x5f46b02a, 0x2b092801, + 0x3d38f5f7, 0x0ca81f36, 0x52af4a8a, 0x66d5e7c0, 0xdf3b0874, 0x95055110, 0x1b5ad7a8, 0xf61ed5ad, + 0x6cf6e479, 0x20758184, 0xd0cefa65, 0x88f7be58, 0x4a046826, 0x0ff6f8f3, 0xa09c7f70, 0x5346aba0, + 0x5ce96c28, 0xe176eda3, 0x6bac307f, 0x376829d2, 0x85360fa9, 0x17e3fe2a, 0x24b79767, 0xf5a96b20, + 0xd6cd2595, 0x68ff1ebf, 0x7555442c, 0xf19f06be, 0xf9e0659a, 0xeeb9491d, 0x34010718, 0xbb30cab8, + 0xe822fe15, 0x88570983, 0x750e6249, 0xda627e55, 0x5e76ffa8, 0xb1534546, 0x6d47de08, 0xefe9e7d4, + }, + { + 0xf6fa8f9d, 0x2cac6ce1, 0x4ca34867, 0xe2337f7c, 0x95db08e7, 0x016843b4, 0xeced5cbc, 0x325553ac, + 0xbf9f0960, 0xdfa1e2ed, 0x83f0579d, 0x63ed86b9, 0x1ab6a6b8, 0xde5ebe39, 0xf38ff732, 0x8989b138, + 0x33f14961, 0xc01937bd, 0xf506c6da, 0xe4625e7e, 0xa308ea99, 0x4e23e33c, 0x79cbd7cc, 0x48a14367, + 0xa3149619, 0xfec94bd5, 0xa114174a, 0xeaa01866, 0xa084db2d, 0x09a8486f, 0xa888614a, 0x2900af98, + 0x01665991, 0xe1992863, 0xc8f30c60, 0x2e78ef3c, 0xd0d51932, 0xcf0fec14, 0xf7ca07d2, 0xd0a82072, + 0xfd41197e, 0x9305a6b0, 0xe86be3da, 0x74bed3cd, 0x372da53c, 0x4c7f4448, 0xdab5d440, 0x6dba0ec3, + 0x083919a7, 0x9fbaeed9, 0x49dbcfb0, 0x4e670c53, 0x5c3d9c01, 0x64bdb941, 0x2c0e636a, 0xba7dd9cd, + 0xea6f7388, 0xe70bc762, 0x35f29adb, 0x5c4cdd8d, 0xf0d48d8c, 0xb88153e2, 0x08a19866, 0x1ae2eac8, + 0x284caf89, 0xaa928223, 0x9334be53, 0x3b3a21bf, 0x16434be3, 0x9aea3906, 0xefe8c36e, 0xf890cdd9, + 0x80226dae, 0xc340a4a3, 0xdf7e9c09, 0xa694a807, 0x5b7c5ecc, 0x221db3a6, 0x9a69a02f, 0x68818a54, + 0xceb2296f, 0x53c0843a, 0xfe893655, 0x25bfe68a, 0xb4628abc, 0xcf222ebf, 0x25ac6f48, 0xa9a99387, + 0x53bddb65, 0xe76ffbe7, 0xe967fd78, 0x0ba93563, 0x8e342bc1, 0xe8a11be9, 0x4980740d, 0xc8087dfc, + 0x8de4bf99, 0xa11101a0, 0x7fd37975, 0xda5a26c0, 0xe81f994f, 0x9528cd89, 0xfd339fed, 0xb87834bf, + 0x5f04456d, 0x22258698, 0xc9c4c83b, 0x2dc156be, 0x4f628daa, 0x57f55ec5, 0xe2220abe, 0xd2916ebf, + 0x4ec75b95, 0x24f2c3c0, 0x42d15d99, 0xcd0d7fa0, 0x7b6e27ff, 0xa8dc8af0, 0x7345c106, 0xf41e232f, + 0x35162386, 0xe6ea8926, 0x3333b094, 0x157ec6f2, 0x372b74af, 0x692573e4, 0xe9a9d848, 0xf3160289, + 0x3a62ef1d, 0xa787e238, 0xf3a5f676, 0x74364853, 0x20951063, 0x4576698d, 0xb6fad407, 0x592af950, + 0x36f73523, 0x4cfb6e87, 0x7da4cec0, 0x6c152daa, 0xcb0396a8, 0xc50dfe5d, 0xfcd707ab, 0x0921c42f, + 0x89dff0bb, 0x5fe2be78, 0x448f4f33, 0x754613c9, 0x2b05d08d, 0x48b9d585, 0xdc049441, 0xc8098f9b, + 0x7dede786, 0xc39a3373, 0x42410005, 0x6a091751, 0x0ef3c8a6, 0x890072d6, 0x28207682, 0xa9a9f7be, + 0xbf32679d, 0xd45b5b75, 0xb353fd00, 0xcbb0e358, 0x830f220a, 0x1f8fb214, 0xd372cf08, 0xcc3c4a13, + 0x8cf63166, 0x061c87be, 0x88c98f88, 0x6062e397, 0x47cf8e7a, 0xb6c85283, 0x3cc2acfb, 0x3fc06976, + 0x4e8f0252, 0x64d8314d, 0xda3870e3, 0x1e665459, 0xc10908f0, 0x513021a5, 0x6c5b68b7, 0x822f8aa0, + 0x3007cd3e, 0x74719eef, 0xdc872681, 0x073340d4, 0x7e432fd9, 0x0c5ec241, 0x8809286c, 0xf592d891, + 0x08a930f6, 0x957ef305, 0xb7fbffbd, 0xc266e96f, 0x6fe4ac98, 0xb173ecc0, 0xbc60b42a, 0x953498da, + 0xfba1ae12, 0x2d4bd736, 0x0f25faab, 0xa4f3fceb, 0xe2969123, 0x257f0c3d, 0x9348af49, 0x361400bc, + 0xe8816f4a, 0x3814f200, 0xa3f94043, 0x9c7a54c2, 0xbc704f57, 0xda41e7f9, 0xc25ad33a, 0x54f4a084, + 0xb17f5505, 0x59357cbe, 0xedbd15c8, 0x7f97c5ab, 0xba5ac7b5, 0xb6f6deaf, 0x3a479c3a, 0x5302da25, + 0x653d7e6a, 0x54268d49, 0x51a477ea, 0x5017d55b, 0xd7d25d88, 0x44136c76, 0x0404a8c8, 0xb8e5a121, + 0xb81a928a, 0x60ed5869, 0x97c55b96, 0xeaec991b, 0x29935913, 0x01fdb7f1, 0x088e8dfa, 0x9ab6f6f5, + 0x3b4cbf9f, 0x4a5de3ab, 0xe6051d35, 0xa0e1d855, 0xd36b4cf1, 0xf544edeb, 0xb0e93524, 0xbebb8fbd, + 0xa2d762cf, 0x49c92f54, 0x38b5f331, 0x7128a454, 0x48392905, 0xa65b1db8, 0x851c97bd, 0xd675cf2f, + }, + { + 0x85e04019, 0x332bf567, 0x662dbfff, 0xcfc65693, 0x2a8d7f6f, 0xab9bc912, 0xde6008a1, 0x2028da1f, + 0x0227bce7, 0x4d642916, 0x18fac300, 0x50f18b82, 0x2cb2cb11, 0xb232e75c, 0x4b3695f2, 0xb28707de, + 0xa05fbcf6, 0xcd4181e9, 0xe150210c, 0xe24ef1bd, 0xb168c381, 0xfde4e789, 0x5c79b0d8, 0x1e8bfd43, + 0x4d495001, 0x38be4341, 0x913cee1d, 0x92a79c3f, 0x089766be, 0xbaeeadf4, 0x1286becf, 0xb6eacb19, + 0x2660c200, 0x7565bde4, 0x64241f7a, 0x8248dca9, 0xc3b3ad66, 0x28136086, 0x0bd8dfa8, 0x356d1cf2, + 0x107789be, 0xb3b2e9ce, 0x0502aa8f, 0x0bc0351e, 0x166bf52a, 0xeb12ff82, 0xe3486911, 0xd34d7516, + 0x4e7b3aff, 0x5f43671b, 0x9cf6e037, 0x4981ac83, 0x334266ce, 0x8c9341b7, 0xd0d854c0, 0xcb3a6c88, + 0x47bc2829, 0x4725ba37, 0xa66ad22b, 0x7ad61f1e, 0x0c5cbafa, 0x4437f107, 0xb6e79962, 0x42d2d816, + 0x0a961288, 0xe1a5c06e, 0x13749e67, 0x72fc081a, 0xb1d139f7, 0xf9583745, 0xcf19df58, 0xbec3f756, + 0xc06eba30, 0x07211b24, 0x45c28829, 0xc95e317f, 0xbc8ec511, 0x38bc46e9, 0xc6e6fa14, 0xbae8584a, + 0xad4ebc46, 0x468f508b, 0x7829435f, 0xf124183b, 0x821dba9f, 0xaff60ff4, 0xea2c4e6d, 0x16e39264, + 0x92544a8b, 0x009b4fc3, 0xaba68ced, 0x9ac96f78, 0x06a5b79a, 0xb2856e6e, 0x1aec3ca9, 0xbe838688, + 0x0e0804e9, 0x55f1be56, 0xe7e5363b, 0xb3a1f25d, 0xf7debb85, 0x61fe033c, 0x16746233, 0x3c034c28, + 0xda6d0c74, 0x79aac56c, 0x3ce4e1ad, 0x51f0c802, 0x98f8f35a, 0x1626a49f, 0xeed82b29, 0x1d382fe3, + 0x0c4fb99a, 0xbb325778, 0x3ec6d97b, 0x6e77a6a9, 0xcb658b5c, 0xd45230c7, 0x2bd1408b, 0x60c03eb7, + 0xb9068d78, 0xa33754f4, 0xf430c87d, 0xc8a71302, 0xb96d8c32, 0xebd4e7be, 0xbe8b9d2d, 0x7979fb06, + 0xe7225308, 0x8b75cf77, 0x11ef8da4, 0xe083c858, 0x8d6b786f, 0x5a6317a6, 0xfa5cf7a0, 0x5dda0033, + 0xf28ebfb0, 0xf5b9c310, 0xa0eac280, 0x08b9767a, 0xa3d9d2b0, 0x79d34217, 0x021a718d, 0x9ac6336a, + 0x2711fd60, 0x438050e3, 0x069908a8, 0x3d7fedc4, 0x826d2bef, 0x4eeb8476, 0x488dcf25, 0x36c9d566, + 0x28e74e41, 0xc2610aca, 0x3d49a9cf, 0xbae3b9df, 0xb65f8de6, 0x92aeaf64, 0x3ac7d5e6, 0x9ea80509, + 0xf22b017d, 0xa4173f70, 0xdd1e16c3, 0x15e0d7f9, 0x50b1b887, 0x2b9f4fd5, 0x625aba82, 0x6a017962, + 0x2ec01b9c, 0x15488aa9, 0xd716e740, 0x40055a2c, 0x93d29a22, 0xe32dbf9a, 0x058745b9, 0x3453dc1e, + 0xd699296e, 0x496cff6f, 0x1c9f4986, 0xdfe2ed07, 0xb87242d1, 0x19de7eae, 0x053e561a, 0x15ad6f8c, + 0x66626c1c, 0x7154c24c, 0xea082b2a, 0x93eb2939, 0x17dcb0f0, 0x58d4f2ae, 0x9ea294fb, 0x52cf564c, + 0x9883fe66, 0x2ec40581, 0x763953c3, 0x01d6692e, 0xd3a0c108, 0xa1e7160e, 0xe4f2dfa6, 0x693ed285, + 0x74904698, 0x4c2b0edd, 0x4f757656, 0x5d393378, 0xa132234f, 0x3d321c5d, 0xc3f5e194, 0x4b269301, + 0xc79f022f, 0x3c997e7e, 0x5e4f9504, 0x3ffafbbd, 0x76f7ad0e, 0x296693f4, 0x3d1fce6f, 0xc61e45be, + 0xd3b5ab34, 0xf72bf9b7, 0x1b0434c0, 0x4e72b567, 0x5592a33d, 0xb5229301, 0xcfd2a87f, 0x60aeb767, + 0x1814386b, 0x30bcc33d, 0x38a0c07d, 0xfd1606f2, 0xc363519b, 0x589dd390, 0x5479f8e6, 0x1cb8d647, + 0x97fd61a9, 0xea7759f4, 0x2d57539d, 0x569a58cf, 0xe84e63ad, 0x462e1b78, 0x6580f87e, 0xf3817914, + 0x91da55f4, 0x40a230f3, 0xd1988f35, 0xb6e318d2, 0x3ffa50bc, 0x3d40f021, 0xc3c0bdae, 0x4958c24c, + 0x518f36b2, 0x84b1d370, 0x0fedce83, 0x878ddada, 0xf2a279c7, 0x94e01be8, 0x90716f4b, 0x954b8aa3, + }, + { + 0xe216300d, 0xbbddfffc, 0xa7ebdabd, 0x35648095, 0x7789f8b7, 0xe6c1121b, 0x0e241600, 0x052ce8b5, + 0x11a9cfb0, 0xe5952f11, 0xece7990a, 0x9386d174, 0x2a42931c, 0x76e38111, 0xb12def3a, 0x37ddddfc, + 0xde9adeb1, 0x0a0cc32c, 0xbe197029, 0x84a00940, 0xbb243a0f, 0xb4d137cf, 0xb44e79f0, 0x049eedfd, + 0x0b15a15d, 0x480d3168, 0x8bbbde5a, 0x669ded42, 0xc7ece831, 0x3f8f95e7, 0x72df191b, 0x7580330d, + 0x94074251, 0x5c7dcdfa, 0xabbe6d63, 0xaa402164, 0xb301d40a, 0x02e7d1ca, 0x53571dae, 0x7a3182a2, + 0x12a8ddec, 0xfdaa335d, 0x176f43e8, 0x71fb46d4, 0x38129022, 0xce949ad4, 0xb84769ad, 0x965bd862, + 0x82f3d055, 0x66fb9767, 0x15b80b4e, 0x1d5b47a0, 0x4cfde06f, 0xc28ec4b8, 0x57e8726e, 0x647a78fc, + 0x99865d44, 0x608bd593, 0x6c200e03, 0x39dc5ff6, 0x5d0b00a3, 0xae63aff2, 0x7e8bd632, 0x70108c0c, + 0xbbd35049, 0x2998df04, 0x980cf42a, 0x9b6df491, 0x9e7edd53, 0x06918548, 0x58cb7e07, 0x3b74ef2e, + 0x522fffb1, 0xd24708cc, 0x1c7e27cd, 0xa4eb215b, 0x3cf1d2e2, 0x19b47a38, 0x424f7618, 0x35856039, + 0x9d17dee7, 0x27eb35e6, 0xc9aff67b, 0x36baf5b8, 0x09c467cd, 0xc18910b1, 0xe11dbf7b, 0x06cd1af8, + 0x7170c608, 0x2d5e3354, 0xd4de495a, 0x64c6d006, 0xbcc0c62c, 0x3dd00db3, 0x708f8f34, 0x77d51b42, + 0x264f620f, 0x24b8d2bf, 0x15c1b79e, 0x46a52564, 0xf8d7e54e, 0x3e378160, 0x7895cda5, 0x859c15a5, + 0xe6459788, 0xc37bc75f, 0xdb07ba0c, 0x0676a3ab, 0x7f229b1e, 0x31842e7b, 0x24259fd7, 0xf8bef472, + 0x835ffcb8, 0x6df4c1f2, 0x96f5b195, 0xfd0af0fc, 0xb0fe134c, 0xe2506d3d, 0x4f9b12ea, 0xf215f225, + 0xa223736f, 0x9fb4c428, 0x25d04979, 0x34c713f8, 0xc4618187, 0xea7a6e98, 0x7cd16efc, 0x1436876c, + 0xf1544107, 0xbedeee14, 0x56e9af27, 0xa04aa441, 0x3cf7c899, 0x92ecbae6, 0xdd67016d, 0x151682eb, + 0xa842eedf, 0xfdba60b4, 0xf1907b75, 0x20e3030f, 0x24d8c29e, 0xe139673b, 0xefa63fb8, 0x71873054, + 0xb6f2cf3b, 0x9f326442, 0xcb15a4cc, 0xb01a4504, 0xf1e47d8d, 0x844a1be5, 0xbae7dfdc, 0x42cbda70, + 0xcd7dae0a, 0x57e85b7a, 0xd53f5af6, 0x20cf4d8c, 0xcea4d428, 0x79d130a4, 0x3486ebfb, 0x33d3cddc, + 0x77853b53, 0x37effcb5, 0xc5068778, 0xe580b3e6, 0x4e68b8f4, 0xc5c8b37e, 0x0d809ea2, 0x398feb7c, + 0x132a4f94, 0x43b7950e, 0x2fee7d1c, 0x223613bd, 0xdd06caa2, 0x37df932b, 0xc4248289, 0xacf3ebc3, + 0x5715f6b7, 0xef3478dd, 0xf267616f, 0xc148cbe4, 0x9052815e, 0x5e410fab, 0xb48a2465, 0x2eda7fa4, + 0xe87b40e4, 0xe98ea084, 0x5889e9e1, 0xefd390fc, 0xdd07d35b, 0xdb485694, 0x38d7e5b2, 0x57720101, + 0x730edebc, 0x5b643113, 0x94917e4f, 0x503c2fba, 0x646f1282, 0x7523d24a, 0xe0779695, 0xf9c17a8f, + 0x7a5b2121, 0xd187b896, 0x29263a4d, 0xba510cdf, 0x81f47c9f, 0xad1163ed, 0xea7b5965, 0x1a00726e, + 0x11403092, 0x00da6d77, 0x4a0cdd61, 0xad1f4603, 0x605bdfb0, 0x9eedc364, 0x22ebe6a8, 0xcee7d28a, + 0xa0e736a0, 0x5564a6b9, 0x10853209, 0xc7eb8f37, 0x2de705ca, 0x8951570f, 0xdf09822b, 0xbd691a6c, + 0xaa12e4f2, 0x87451c0f, 0xe0f6a27a, 0x3ada4819, 0x4cf1764f, 0x0d771c2b, 0x67cdb156, 0x350d8384, + 0x5938fa0f, 0x42399ef3, 0x36997b07, 0x0e84093d, 0x4aa93e61, 0x8360d87b, 0x1fa98b0c, 0x1149382c, + 0xe97625a5, 0x0614d1b7, 0x0e25244b, 0x0c768347, 0x589e8d82, 0x0d2059d1, 0xa466bb1e, 0xf8da0a82, + 0x04f19130, 0xba6e4ec0, 0x99265164, 0x1ee7230d, 0x50b2ad80, 0xeaee6801, 0x8db2a283, 0xea8bf59e, + }, +} diff --git a/libgo/go/crypto/cast5/cast5_test.go b/libgo/go/crypto/cast5/cast5_test.go new file mode 100644 index 000000000..5f7025ff2 --- /dev/null +++ b/libgo/go/crypto/cast5/cast5_test.go @@ -0,0 +1,104 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cast5 + +import ( + "bytes" + "encoding/hex" + "testing" +) + +// This test vector is taken from RFC 2144, App B.1. +// Since the other two test vectors are for reduced-round variants, we can't +// use them. +var basicTests = []struct { + key, plainText, cipherText string +}{ + { + "0123456712345678234567893456789a", + "0123456789abcdef", + "238b4fe5847e44b2", + }, +} + +func TestBasic(t *testing.T) { + for i, test := range basicTests { + key, _ := hex.DecodeString(test.key) + plainText, _ := hex.DecodeString(test.plainText) + expected, _ := hex.DecodeString(test.cipherText) + + c, err := NewCipher(key) + if err != nil { + t.Errorf("#%d: failed to create Cipher: %s", i, err) + continue + } + var cipherText [BlockSize]byte + c.Encrypt(cipherText[:], plainText) + if !bytes.Equal(cipherText[:], expected) { + t.Errorf("#%d: got:%x want:%x", i, cipherText, expected) + } + + var plainTextAgain [BlockSize]byte + c.Decrypt(plainTextAgain[:], cipherText[:]) + if !bytes.Equal(plainTextAgain[:], plainText) { + t.Errorf("#%d: got:%x want:%x", i, plainTextAgain, plainText) + } + } +} + +// TestFull performs the test specified in RFC 2144, App B.2. +// However, due to the length of time taken, it's disabled here and a more +// limited version is included, below. +func TestFull(t *testing.T) { + // This is too slow for normal testing + return + + a, b := iterate(1000000) + + const expectedA = "eea9d0a249fd3ba6b3436fb89d6dca92" + const expectedB = "b2c95eb00c31ad7180ac05b8e83d696e" + + if hex.EncodeToString(a) != expectedA { + t.Errorf("a: got:%x want:%s", a, expectedA) + } + if hex.EncodeToString(b) != expectedB { + t.Errorf("b: got:%x want:%s", b, expectedB) + } +} + +func iterate(iterations int) ([]byte, []byte) { + const initValueHex = "0123456712345678234567893456789a" + + initValue, _ := hex.DecodeString(initValueHex) + + var a, b [16]byte + copy(a[:], initValue) + copy(b[:], initValue) + + for i := 0; i < iterations; i++ { + c, _ := NewCipher(b[:]) + c.Encrypt(a[:8], a[:8]) + c.Encrypt(a[8:], a[8:]) + c, _ = NewCipher(a[:]) + c.Encrypt(b[:8], b[:8]) + c.Encrypt(b[8:], b[8:]) + } + + return a[:], b[:] +} + +func TestLimited(t *testing.T) { + a, b := iterate(1000) + + const expectedA = "23f73b14b02a2ad7dfb9f2c35644798d" + const expectedB = "e5bf37eff14c456a40b21ce369370a9f" + + if hex.EncodeToString(a) != expectedA { + t.Errorf("a: got:%x want:%s", a, expectedA) + } + if hex.EncodeToString(b) != expectedB { + t.Errorf("b: got:%x want:%s", b, expectedB) + } +} diff --git a/libgo/go/crypto/cipher/cbc.go b/libgo/go/crypto/cipher/cbc.go new file mode 100644 index 000000000..4632f882a --- /dev/null +++ b/libgo/go/crypto/cipher/cbc.go @@ -0,0 +1,78 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Cipher block chaining (CBC) mode. + +// CBC provides confidentiality by xoring (chaining) each plaintext block +// with the previous ciphertext block before applying the block cipher. + +// See NIST SP 800-38A, pp 10-11 + +package cipher + +type cbc struct { + b Block + blockSize int + iv []byte + tmp []byte +} + +func newCBC(b Block, iv []byte) *cbc { + return &cbc{ + b: b, + blockSize: b.BlockSize(), + iv: dup(iv), + tmp: make([]byte, b.BlockSize()), + } +} + +type cbcEncrypter cbc + +// NewCBCEncrypter returns a BlockMode which encrypts in cipher block chaining +// mode, using the given Block. The length of iv must be the same as the +// Block's block size. +func NewCBCEncrypter(b Block, iv []byte) BlockMode { + return (*cbcEncrypter)(newCBC(b, iv)) +} + +func (x *cbcEncrypter) BlockSize() int { return x.blockSize } + +func (x *cbcEncrypter) CryptBlocks(dst, src []byte) { + for len(src) > 0 { + for i := 0; i < x.blockSize; i++ { + x.iv[i] ^= src[i] + } + x.b.Encrypt(x.iv, x.iv) + for i := 0; i < x.blockSize; i++ { + dst[i] = x.iv[i] + } + src = src[x.blockSize:] + dst = dst[x.blockSize:] + } +} + +type cbcDecrypter cbc + +// NewCBCDecrypter returns a BlockMode which decrypts in cipher block chaining +// mode, using the given Block. The length of iv must be the same as the +// Block's block size as must match the iv used to encrypt the data. +func NewCBCDecrypter(b Block, iv []byte) BlockMode { + return (*cbcDecrypter)(newCBC(b, iv)) +} + +func (x *cbcDecrypter) BlockSize() int { return x.blockSize } + +func (x *cbcDecrypter) CryptBlocks(dst, src []byte) { + for len(src) > 0 { + x.b.Decrypt(x.tmp, src[:x.blockSize]) + for i := 0; i < x.blockSize; i++ { + x.tmp[i] ^= x.iv[i] + x.iv[i] = src[i] + dst[i] = x.tmp[i] + } + + src = src[x.blockSize:] + dst = dst[x.blockSize:] + } +} diff --git a/libgo/go/crypto/cipher/cbc_aes_test.go b/libgo/go/crypto/cipher/cbc_aes_test.go new file mode 100644 index 000000000..944ca1ba8 --- /dev/null +++ b/libgo/go/crypto/cipher/cbc_aes_test.go @@ -0,0 +1,89 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// CBC AES test vectors. + +// See U.S. National Institute of Standards and Technology (NIST) +// Special Publication 800-38A, ``Recommendation for Block Cipher +// Modes of Operation,'' 2001 Edition, pp. 24-29. + +package cipher + +import ( + "bytes" + "crypto/aes" + "testing" +) + +var cbcAESTests = []struct { + name string + key []byte + iv []byte + in []byte + out []byte +}{ + // NIST SP 800-38A pp 27-29 + { + "CBC-AES128", + commonKey128, + commonIV, + commonInput, + []byte{ + 0x76, 0x49, 0xab, 0xac, 0x81, 0x19, 0xb2, 0x46, 0xce, 0xe9, 0x8e, 0x9b, 0x12, 0xe9, 0x19, 0x7d, + 0x50, 0x86, 0xcb, 0x9b, 0x50, 0x72, 0x19, 0xee, 0x95, 0xdb, 0x11, 0x3a, 0x91, 0x76, 0x78, 0xb2, + 0x73, 0xbe, 0xd6, 0xb8, 0xe3, 0xc1, 0x74, 0x3b, 0x71, 0x16, 0xe6, 0x9e, 0x22, 0x22, 0x95, 0x16, + 0x3f, 0xf1, 0xca, 0xa1, 0x68, 0x1f, 0xac, 0x09, 0x12, 0x0e, 0xca, 0x30, 0x75, 0x86, 0xe1, 0xa7, + }, + }, + { + "CBC-AES192", + commonKey192, + commonIV, + commonInput, + []byte{ + 0x4f, 0x02, 0x1d, 0xb2, 0x43, 0xbc, 0x63, 0x3d, 0x71, 0x78, 0x18, 0x3a, 0x9f, 0xa0, 0x71, 0xe8, + 0xb4, 0xd9, 0xad, 0xa9, 0xad, 0x7d, 0xed, 0xf4, 0xe5, 0xe7, 0x38, 0x76, 0x3f, 0x69, 0x14, 0x5a, + 0x57, 0x1b, 0x24, 0x20, 0x12, 0xfb, 0x7a, 0xe0, 0x7f, 0xa9, 0xba, 0xac, 0x3d, 0xf1, 0x02, 0xe0, + 0x08, 0xb0, 0xe2, 0x79, 0x88, 0x59, 0x88, 0x81, 0xd9, 0x20, 0xa9, 0xe6, 0x4f, 0x56, 0x15, 0xcd, + }, + }, + { + "CBC-AES256", + commonKey256, + commonIV, + commonInput, + []byte{ + 0xf5, 0x8c, 0x4c, 0x04, 0xd6, 0xe5, 0xf1, 0xba, 0x77, 0x9e, 0xab, 0xfb, 0x5f, 0x7b, 0xfb, 0xd6, + 0x9c, 0xfc, 0x4e, 0x96, 0x7e, 0xdb, 0x80, 0x8d, 0x67, 0x9f, 0x77, 0x7b, 0xc6, 0x70, 0x2c, 0x7d, + 0x39, 0xf2, 0x33, 0x69, 0xa9, 0xd9, 0xba, 0xcf, 0xa5, 0x30, 0xe2, 0x63, 0x04, 0x23, 0x14, 0x61, + 0xb2, 0xeb, 0x05, 0xe2, 0xc3, 0x9b, 0xe9, 0xfc, 0xda, 0x6c, 0x19, 0x07, 0x8c, 0x6a, 0x9d, 0x1b, + }, + }, +} + +func TestCBC_AES(t *testing.T) { + for _, tt := range cbcAESTests { + test := tt.name + + c, err := aes.NewCipher(tt.key) + if err != nil { + t.Errorf("%s: NewCipher(%d bytes) = %s", test, len(tt.key), err) + continue + } + + encrypter := NewCBCEncrypter(c, tt.iv) + d := make([]byte, len(tt.in)) + encrypter.CryptBlocks(d, tt.in) + if !bytes.Equal(tt.out, d) { + t.Errorf("%s: CBCEncrypter\nhave %x\nwant %x", test, d, tt.out) + } + + decrypter := NewCBCDecrypter(c, tt.iv) + p := make([]byte, len(d)) + decrypter.CryptBlocks(p, d) + if !bytes.Equal(tt.in, p) { + t.Errorf("%s: CBCDecrypter\nhave %x\nwant %x", test, d, tt.in) + } + } +} diff --git a/libgo/go/crypto/cipher/cfb.go b/libgo/go/crypto/cipher/cfb.go new file mode 100644 index 000000000..d14165a86 --- /dev/null +++ b/libgo/go/crypto/cipher/cfb.go @@ -0,0 +1,64 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// CFB (Cipher Feedback) Mode. + +package cipher + +type cfb struct { + b Block + out []byte + outUsed int + decrypt bool +} + +// NewCFBEncrypter returns a Stream which encrypts with cipher feedback mode, +// using the given Block. The iv must be the same length as the Block's block +// size. +func NewCFBEncrypter(block Block, iv []byte) Stream { + return newCFB(block, iv, false) +} + +// NewCFBDecrypter returns a Stream which decrypts with cipher feedback mode, +// using the given Block. The iv must be the same length as the Block's block +// size. +func NewCFBDecrypter(block Block, iv []byte) Stream { + return newCFB(block, iv, true) +} + +func newCFB(block Block, iv []byte, decrypt bool) Stream { + blockSize := block.BlockSize() + if len(iv) != blockSize { + return nil + } + + x := &cfb{ + b: block, + out: make([]byte, blockSize), + outUsed: 0, + decrypt: decrypt, + } + block.Encrypt(x.out, iv) + + return x +} + +func (x *cfb) XORKeyStream(dst, src []byte) { + for i := 0; i < len(src); i++ { + if x.outUsed == len(x.out) { + x.b.Encrypt(x.out, x.out) + x.outUsed = 0 + } + + if x.decrypt { + t := src[i] + dst[i] = src[i] ^ x.out[x.outUsed] + x.out[x.outUsed] = t + } else { + x.out[x.outUsed] ^= src[i] + dst[i] = x.out[x.outUsed] + } + x.outUsed++ + } +} diff --git a/libgo/go/crypto/cipher/cfb_test.go b/libgo/go/crypto/cipher/cfb_test.go new file mode 100644 index 000000000..9547bfceb --- /dev/null +++ b/libgo/go/crypto/cipher/cfb_test.go @@ -0,0 +1,35 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cipher + +import ( + "bytes" + "crypto/aes" + "crypto/rand" + "testing" +) + +func TestCFB(t *testing.T) { + block, err := aes.NewCipher(commonKey128) + if err != nil { + t.Error(err) + return + } + + plaintext := []byte("this is the plaintext") + iv := make([]byte, block.BlockSize()) + rand.Reader.Read(iv) + cfb := NewCFBEncrypter(block, iv) + ciphertext := make([]byte, len(plaintext)) + cfb.XORKeyStream(ciphertext, plaintext) + + cfbdec := NewCFBDecrypter(block, iv) + plaintextCopy := make([]byte, len(plaintext)) + cfbdec.XORKeyStream(plaintextCopy, ciphertext) + + if !bytes.Equal(plaintextCopy, plaintext) { + t.Errorf("got: %x, want: %x", plaintextCopy, plaintext) + } +} diff --git a/libgo/go/crypto/cipher/cipher.go b/libgo/go/crypto/cipher/cipher.go new file mode 100644 index 000000000..50516b23a --- /dev/null +++ b/libgo/go/crypto/cipher/cipher.go @@ -0,0 +1,63 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The cipher package implements standard block cipher modes +// that can be wrapped around low-level block cipher implementations. +// See http://csrc.nist.gov/groups/ST/toolkit/BCM/current_modes.html +// and NIST Special Publication 800-38A. +package cipher + +// A Block represents an implementation of block cipher +// using a given key. It provides the capability to encrypt +// or decrypt individual blocks. The mode implementations +// extend that capability to streams of blocks. +type Block interface { + // BlockSize returns the cipher's block size. + BlockSize() int + + // Encrypt encrypts the first block in src into dst. + // Dst and src may point at the same memory. + Encrypt(dst, src []byte) + + // Decrypt decrypts the first block in src into dst. + // Dst and src may point at the same memory. + Decrypt(dst, src []byte) +} + +// A Stream represents a stream cipher. +type Stream interface { + // XORKeyStream XORs each byte in the given slice with a byte from the + // cipher's key stream. Dst and src may point to the same memory. + XORKeyStream(dst, src []byte) +} + +// A BlockMode represents a block cipher running in a block-based mode (CBC, +// ECB etc). +type BlockMode interface { + // BlockSize returns the mode's block size. + BlockSize() int + + // CryptBlocks encrypts or decrypts a number of blocks. The length of + // src must be a multiple of the block size. Dst and src may point to + // the same memory. + CryptBlocks(dst, src []byte) +} + +// Utility routines + +func shift1(dst, src []byte) byte { + var b byte + for i := len(src) - 1; i >= 0; i-- { + bb := src[i] >> 7 + dst[i] = src[i]<<1 | b + b = bb + } + return b +} + +func dup(p []byte) []byte { + q := make([]byte, len(p)) + copy(q, p) + return q +} diff --git a/libgo/go/crypto/cipher/common_test.go b/libgo/go/crypto/cipher/common_test.go new file mode 100644 index 000000000..fb755757c --- /dev/null +++ b/libgo/go/crypto/cipher/common_test.go @@ -0,0 +1,28 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cipher + +// Common values for tests. + +var commonInput = []byte{ + 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a, + 0xae, 0x2d, 0x8a, 0x57, 0x1e, 0x03, 0xac, 0x9c, 0x9e, 0xb7, 0x6f, 0xac, 0x45, 0xaf, 0x8e, 0x51, + 0x30, 0xc8, 0x1c, 0x46, 0xa3, 0x5c, 0xe4, 0x11, 0xe5, 0xfb, 0xc1, 0x19, 0x1a, 0x0a, 0x52, 0xef, + 0xf6, 0x9f, 0x24, 0x45, 0xdf, 0x4f, 0x9b, 0x17, 0xad, 0x2b, 0x41, 0x7b, 0xe6, 0x6c, 0x37, 0x10, +} + +var commonKey128 = []byte{0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c} + +var commonKey192 = []byte{ + 0x8e, 0x73, 0xb0, 0xf7, 0xda, 0x0e, 0x64, 0x52, 0xc8, 0x10, 0xf3, 0x2b, 0x80, 0x90, 0x79, 0xe5, + 0x62, 0xf8, 0xea, 0xd2, 0x52, 0x2c, 0x6b, 0x7b, +} + +var commonKey256 = []byte{ + 0x60, 0x3d, 0xeb, 0x10, 0x15, 0xca, 0x71, 0xbe, 0x2b, 0x73, 0xae, 0xf0, 0x85, 0x7d, 0x77, 0x81, + 0x1f, 0x35, 0x2c, 0x07, 0x3b, 0x61, 0x08, 0xd7, 0x2d, 0x98, 0x10, 0xa3, 0x09, 0x14, 0xdf, 0xf4, +} + +var commonIV = []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f} diff --git a/libgo/go/crypto/cipher/ctr.go b/libgo/go/crypto/cipher/ctr.go new file mode 100644 index 000000000..04436ec23 --- /dev/null +++ b/libgo/go/crypto/cipher/ctr.go @@ -0,0 +1,51 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Counter (CTR) mode. + +// CTR converts a block cipher into a stream cipher by +// repeatedly encrypting an incrementing counter and +// xoring the resulting stream of data with the input. + +// See NIST SP 800-38A, pp 13-15 + +package cipher + +type ctr struct { + b Block + ctr []byte + out []byte + outUsed int +} + +// NewCTR returns a Stream which encrypts/decrypts using the given Block in +// counter mode. The length of iv must be the same as the Block's block size. +func NewCTR(block Block, iv []byte) Stream { + return &ctr{ + b: block, + ctr: dup(iv), + out: make([]byte, len(iv)), + outUsed: len(iv), + } +} + +func (x *ctr) XORKeyStream(dst, src []byte) { + for i := 0; i < len(src); i++ { + if x.outUsed == len(x.ctr) { + x.b.Encrypt(x.out, x.ctr) + x.outUsed = 0 + + // Increment counter + for i := len(x.ctr) - 1; i >= 0; i-- { + x.ctr[i]++ + if x.ctr[i] != 0 { + break + } + } + } + + dst[i] = src[i] ^ x.out[x.outUsed] + x.outUsed++ + } +} diff --git a/libgo/go/crypto/cipher/ctr_aes_test.go b/libgo/go/crypto/cipher/ctr_aes_test.go new file mode 100644 index 000000000..8dca9968c --- /dev/null +++ b/libgo/go/crypto/cipher/ctr_aes_test.go @@ -0,0 +1,101 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// CTR AES test vectors. + +// See U.S. National Institute of Standards and Technology (NIST) +// Special Publication 800-38A, ``Recommendation for Block Cipher +// Modes of Operation,'' 2001 Edition, pp. 55-58. + +package cipher + +import ( + "bytes" + "crypto/aes" + "testing" +) + +var commonCounter = []byte{0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff} + +var ctrAESTests = []struct { + name string + key []byte + iv []byte + in []byte + out []byte +}{ + // NIST SP 800-38A pp 55-58 + { + "CTR-AES128", + commonKey128, + commonCounter, + commonInput, + []byte{ + 0x87, 0x4d, 0x61, 0x91, 0xb6, 0x20, 0xe3, 0x26, 0x1b, 0xef, 0x68, 0x64, 0x99, 0x0d, 0xb6, 0xce, + 0x98, 0x06, 0xf6, 0x6b, 0x79, 0x70, 0xfd, 0xff, 0x86, 0x17, 0x18, 0x7b, 0xb9, 0xff, 0xfd, 0xff, + 0x5a, 0xe4, 0xdf, 0x3e, 0xdb, 0xd5, 0xd3, 0x5e, 0x5b, 0x4f, 0x09, 0x02, 0x0d, 0xb0, 0x3e, 0xab, + 0x1e, 0x03, 0x1d, 0xda, 0x2f, 0xbe, 0x03, 0xd1, 0x79, 0x21, 0x70, 0xa0, 0xf3, 0x00, 0x9c, 0xee, + }, + }, + { + "CTR-AES192", + commonKey192, + commonCounter, + commonInput, + []byte{ + 0x1a, 0xbc, 0x93, 0x24, 0x17, 0x52, 0x1c, 0xa2, 0x4f, 0x2b, 0x04, 0x59, 0xfe, 0x7e, 0x6e, 0x0b, + 0x09, 0x03, 0x39, 0xec, 0x0a, 0xa6, 0xfa, 0xef, 0xd5, 0xcc, 0xc2, 0xc6, 0xf4, 0xce, 0x8e, 0x94, + 0x1e, 0x36, 0xb2, 0x6b, 0xd1, 0xeb, 0xc6, 0x70, 0xd1, 0xbd, 0x1d, 0x66, 0x56, 0x20, 0xab, 0xf7, + 0x4f, 0x78, 0xa7, 0xf6, 0xd2, 0x98, 0x09, 0x58, 0x5a, 0x97, 0xda, 0xec, 0x58, 0xc6, 0xb0, 0x50, + }, + }, + { + "CTR-AES256", + commonKey256, + commonCounter, + commonInput, + []byte{ + 0x60, 0x1e, 0xc3, 0x13, 0x77, 0x57, 0x89, 0xa5, 0xb7, 0xa7, 0xf5, 0x04, 0xbb, 0xf3, 0xd2, 0x28, + 0xf4, 0x43, 0xe3, 0xca, 0x4d, 0x62, 0xb5, 0x9a, 0xca, 0x84, 0xe9, 0x90, 0xca, 0xca, 0xf5, 0xc5, + 0x2b, 0x09, 0x30, 0xda, 0xa2, 0x3d, 0xe9, 0x4c, 0xe8, 0x70, 0x17, 0xba, 0x2d, 0x84, 0x98, 0x8d, + 0xdf, 0xc9, 0xc5, 0x8d, 0xb6, 0x7a, 0xad, 0xa6, 0x13, 0xc2, 0xdd, 0x08, 0x45, 0x79, 0x41, 0xa6, + }, + }, +} + +func TestCTR_AES(t *testing.T) { + for _, tt := range ctrAESTests { + test := tt.name + + c, err := aes.NewCipher(tt.key) + if err != nil { + t.Errorf("%s: NewCipher(%d bytes) = %s", test, len(tt.key), err) + continue + } + + for j := 0; j <= 5; j += 5 { + in := tt.in[0 : len(tt.in)-j] + ctr := NewCTR(c, tt.iv) + encrypted := make([]byte, len(in)) + ctr.XORKeyStream(encrypted, in) + if out := tt.out[0:len(in)]; !bytes.Equal(out, encrypted) { + t.Errorf("%s/%d: CTR\ninpt %x\nhave %x\nwant %x", test, len(in), in, encrypted, out) + } + } + + for j := 0; j <= 7; j += 7 { + in := tt.out[0 : len(tt.out)-j] + ctr := NewCTR(c, tt.iv) + plain := make([]byte, len(in)) + ctr.XORKeyStream(plain, in) + if out := tt.in[0:len(in)]; !bytes.Equal(out, plain) { + t.Errorf("%s/%d: CTRReader\nhave %x\nwant %x", test, len(out), plain, out) + } + } + + if t.Failed() { + break + } + } +} diff --git a/libgo/go/crypto/cipher/io.go b/libgo/go/crypto/cipher/io.go new file mode 100644 index 000000000..97f40b8e7 --- /dev/null +++ b/libgo/go/crypto/cipher/io.go @@ -0,0 +1,57 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cipher + +import ( + "os" + "io" +) + +// The Stream* objects are so simple that all their members are public. Users +// can create them themselves. + +// StreamReader wraps a Stream into an io.Reader. It simply calls XORKeyStream +// to process each slice of data which passes through. +type StreamReader struct { + S Stream + R io.Reader +} + +func (r StreamReader) Read(dst []byte) (n int, err os.Error) { + n, err = r.R.Read(dst) + r.S.XORKeyStream(dst[:n], dst[:n]) + return +} + +// StreamWriter wraps a Stream into an io.Writer. It simply calls XORKeyStream +// to process each slice of data which passes through. If any Write call +// returns short then the StreamWriter is out of sync and must be discarded. +type StreamWriter struct { + S Stream + W io.Writer + Err os.Error +} + +func (w StreamWriter) Write(src []byte) (n int, err os.Error) { + if w.Err != nil { + return 0, w.Err + } + c := make([]byte, len(src)) + w.S.XORKeyStream(c, src) + n, err = w.W.Write(c) + if n != len(src) { + if err == nil { // should never happen + err = io.ErrShortWrite + } + w.Err = err + } + return +} + +func (w StreamWriter) Close() os.Error { + // This saves us from either requiring a WriteCloser or having a + // StreamWriterCloser. + return w.W.(io.Closer).Close() +} diff --git a/libgo/go/crypto/cipher/ocfb.go b/libgo/go/crypto/cipher/ocfb.go new file mode 100644 index 000000000..43cb5a531 --- /dev/null +++ b/libgo/go/crypto/cipher/ocfb.go @@ -0,0 +1,112 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// OpenPGP CFB Mode. http://tools.ietf.org/html/rfc4880#section-13.9 + +package cipher + +type ocfbEncrypter struct { + b Block + fre []byte + outUsed int +} + +// NewOCFBEncrypter returns a Stream which encrypts data with OpenPGP's cipher +// feedback mode using the given Block, and an initial amount of ciphertext. +// randData must be random bytes and be the same length as the Block's block +// size. +func NewOCFBEncrypter(block Block, randData []byte) (Stream, []byte) { + blockSize := block.BlockSize() + if len(randData) != blockSize { + return nil, nil + } + + x := &ocfbEncrypter{ + b: block, + fre: make([]byte, blockSize), + outUsed: 0, + } + prefix := make([]byte, blockSize+2) + + block.Encrypt(x.fre, x.fre) + for i := 0; i < blockSize; i++ { + prefix[i] = randData[i] ^ x.fre[i] + } + + block.Encrypt(x.fre, prefix[:blockSize]) + prefix[blockSize] = x.fre[0] ^ randData[blockSize-2] + prefix[blockSize+1] = x.fre[1] ^ randData[blockSize-1] + + block.Encrypt(x.fre, prefix[2:]) + return x, prefix +} + +func (x *ocfbEncrypter) XORKeyStream(dst, src []byte) { + for i := 0; i < len(src); i++ { + if x.outUsed == len(x.fre) { + x.b.Encrypt(x.fre, x.fre) + x.outUsed = 0 + } + + x.fre[x.outUsed] ^= src[i] + dst[i] = x.fre[x.outUsed] + x.outUsed++ + } +} + +type ocfbDecrypter struct { + b Block + fre []byte + outUsed int +} + +// NewOCFBDecrypter returns a Stream which decrypts data with OpenPGP's cipher +// feedback mode using the given Block. Prefix must be the first blockSize + 2 +// bytes of the ciphertext, where blockSize is the Block's block size. If an +// incorrect key is detected then nil is returned. +func NewOCFBDecrypter(block Block, prefix []byte) Stream { + blockSize := block.BlockSize() + if len(prefix) != blockSize+2 { + return nil + } + + x := &ocfbDecrypter{ + b: block, + fre: make([]byte, blockSize), + outUsed: 0, + } + prefixCopy := make([]byte, len(prefix)) + copy(prefixCopy, prefix) + + block.Encrypt(x.fre, x.fre) + for i := 0; i < blockSize; i++ { + prefixCopy[i] ^= x.fre[i] + } + + block.Encrypt(x.fre, prefix[:blockSize]) + prefixCopy[blockSize] ^= x.fre[0] + prefixCopy[blockSize+1] ^= x.fre[1] + + if prefixCopy[blockSize-2] != prefixCopy[blockSize] || + prefixCopy[blockSize-1] != prefixCopy[blockSize+1] { + return nil + } + + block.Encrypt(x.fre, prefix[2:]) + return x +} + +func (x *ocfbDecrypter) XORKeyStream(dst, src []byte) { + for i := 0; i < len(src); i++ { + if x.outUsed == len(x.fre) { + x.b.Encrypt(x.fre, x.fre) + x.outUsed = 0 + } + + c := src[i] + dst[i] = x.fre[x.outUsed] ^ src[i] + x.fre[x.outUsed] = c + x.outUsed++ + } +} diff --git a/libgo/go/crypto/cipher/ocfb_test.go b/libgo/go/crypto/cipher/ocfb_test.go new file mode 100644 index 000000000..289bb7c91 --- /dev/null +++ b/libgo/go/crypto/cipher/ocfb_test.go @@ -0,0 +1,39 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cipher + +import ( + "bytes" + "crypto/aes" + "crypto/rand" + "testing" +) + +func TestOCFB(t *testing.T) { + block, err := aes.NewCipher(commonKey128) + if err != nil { + t.Error(err) + return + } + + plaintext := []byte("this is the plaintext") + randData := make([]byte, block.BlockSize()) + rand.Reader.Read(randData) + ocfb, prefix := NewOCFBEncrypter(block, randData) + ciphertext := make([]byte, len(plaintext)) + ocfb.XORKeyStream(ciphertext, plaintext) + + ocfbdec := NewOCFBDecrypter(block, prefix) + if ocfbdec == nil { + t.Error("NewOCFBDecrypter failed") + return + } + plaintextCopy := make([]byte, len(plaintext)) + ocfbdec.XORKeyStream(plaintextCopy, ciphertext) + + if !bytes.Equal(plaintextCopy, plaintext) { + t.Errorf("got: %x, want: %x", plaintextCopy, plaintext) + } +} diff --git a/libgo/go/crypto/cipher/ofb.go b/libgo/go/crypto/cipher/ofb.go new file mode 100644 index 000000000..85e5f02b0 --- /dev/null +++ b/libgo/go/crypto/cipher/ofb.go @@ -0,0 +1,44 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// OFB (Output Feedback) Mode. + +package cipher + +type ofb struct { + b Block + out []byte + outUsed int +} + +// NewOFB returns a Stream that encrypts or decrypts using the block cipher b +// in output feedback mode. The initialization vector iv's length must be equal +// to b's block size. +func NewOFB(b Block, iv []byte) Stream { + blockSize := b.BlockSize() + if len(iv) != blockSize { + return nil + } + + x := &ofb{ + b: b, + out: make([]byte, blockSize), + outUsed: 0, + } + b.Encrypt(x.out, iv) + + return x +} + +func (x *ofb) XORKeyStream(dst, src []byte) { + for i, s := range src { + if x.outUsed == len(x.out) { + x.b.Encrypt(x.out, x.out) + x.outUsed = 0 + } + + dst[i] = s ^ x.out[x.outUsed] + x.outUsed++ + } +} diff --git a/libgo/go/crypto/cipher/ofb_test.go b/libgo/go/crypto/cipher/ofb_test.go new file mode 100644 index 000000000..9b4495c88 --- /dev/null +++ b/libgo/go/crypto/cipher/ofb_test.go @@ -0,0 +1,101 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// OFB AES test vectors. + +// See U.S. National Institute of Standards and Technology (NIST) +// Special Publication 800-38A, ``Recommendation for Block Cipher +// Modes of Operation,'' 2001 Edition, pp. 52-55. + +package cipher + +import ( + "bytes" + "crypto/aes" + "testing" +) + +type ofbTest struct { + name string + key []byte + iv []byte + in []byte + out []byte +} + +var ofbTests = []ofbTest{ + // NIST SP 800-38A pp 52-55 + { + "OFB-AES128", + commonKey128, + commonIV, + commonInput, + []byte{ + 0x3b, 0x3f, 0xd9, 0x2e, 0xb7, 0x2d, 0xad, 0x20, 0x33, 0x34, 0x49, 0xf8, 0xe8, 0x3c, 0xfb, 0x4a, + 0x77, 0x89, 0x50, 0x8d, 0x16, 0x91, 0x8f, 0x03, 0xf5, 0x3c, 0x52, 0xda, 0xc5, 0x4e, 0xd8, 0x25, + 0x97, 0x40, 0x05, 0x1e, 0x9c, 0x5f, 0xec, 0xf6, 0x43, 0x44, 0xf7, 0xa8, 0x22, 0x60, 0xed, 0xcc, + 0x30, 0x4c, 0x65, 0x28, 0xf6, 0x59, 0xc7, 0x78, 0x66, 0xa5, 0x10, 0xd9, 0xc1, 0xd6, 0xae, 0x5e, + }, + }, + { + "OFB-AES192", + commonKey192, + commonIV, + commonInput, + []byte{ + 0xcd, 0xc8, 0x0d, 0x6f, 0xdd, 0xf1, 0x8c, 0xab, 0x34, 0xc2, 0x59, 0x09, 0xc9, 0x9a, 0x41, 0x74, + 0xfc, 0xc2, 0x8b, 0x8d, 0x4c, 0x63, 0x83, 0x7c, 0x09, 0xe8, 0x17, 0x00, 0xc1, 0x10, 0x04, 0x01, + 0x8d, 0x9a, 0x9a, 0xea, 0xc0, 0xf6, 0x59, 0x6f, 0x55, 0x9c, 0x6d, 0x4d, 0xaf, 0x59, 0xa5, 0xf2, + 0x6d, 0x9f, 0x20, 0x08, 0x57, 0xca, 0x6c, 0x3e, 0x9c, 0xac, 0x52, 0x4b, 0xd9, 0xac, 0xc9, 0x2a, + }, + }, + { + "OFB-AES256", + commonKey256, + commonIV, + commonInput, + []byte{ + 0xdc, 0x7e, 0x84, 0xbf, 0xda, 0x79, 0x16, 0x4b, 0x7e, 0xcd, 0x84, 0x86, 0x98, 0x5d, 0x38, 0x60, + 0x4f, 0xeb, 0xdc, 0x67, 0x40, 0xd2, 0x0b, 0x3a, 0xc8, 0x8f, 0x6a, 0xd8, 0x2a, 0x4f, 0xb0, 0x8d, + 0x71, 0xab, 0x47, 0xa0, 0x86, 0xe8, 0x6e, 0xed, 0xf3, 0x9d, 0x1c, 0x5b, 0xba, 0x97, 0xc4, 0x08, + 0x01, 0x26, 0x14, 0x1d, 0x67, 0xf3, 0x7b, 0xe8, 0x53, 0x8f, 0x5a, 0x8b, 0xe7, 0x40, 0xe4, 0x84, + }, + }, +} + +func TestOFB(t *testing.T) { + for _, tt := range ofbTests { + test := tt.name + + c, err := aes.NewCipher(tt.key) + if err != nil { + t.Errorf("%s: NewCipher(%d bytes) = %s", test, len(tt.key), err) + continue + } + + for j := 0; j <= 5; j += 5 { + plaintext := tt.in[0 : len(tt.in)-j] + ofb := NewOFB(c, tt.iv) + ciphertext := make([]byte, len(plaintext)) + ofb.XORKeyStream(ciphertext, plaintext) + if !bytes.Equal(ciphertext, tt.out[:len(plaintext)]) { + t.Errorf("%s/%d: encrypting\ninput % x\nhave % x\nwant % x", test, len(plaintext), plaintext, ciphertext, tt.out) + } + } + + for j := 0; j <= 5; j += 5 { + ciphertext := tt.out[0 : len(tt.in)-j] + ofb := NewOFB(c, tt.iv) + plaintext := make([]byte, len(ciphertext)) + ofb.XORKeyStream(plaintext, ciphertext) + if !bytes.Equal(plaintext, tt.in[:len(ciphertext)]) { + t.Errorf("%s/%d: decrypting\nhave % x\nwant % x", test, len(ciphertext), plaintext, tt.in) + } + } + + if t.Failed() { + break + } + } +} diff --git a/libgo/go/crypto/elliptic/elliptic.go b/libgo/go/crypto/elliptic/elliptic.go new file mode 100644 index 000000000..beac45ca0 --- /dev/null +++ b/libgo/go/crypto/elliptic/elliptic.go @@ -0,0 +1,376 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The elliptic package implements several standard elliptic curves over prime +// fields +package elliptic + +// This package operates, internally, on Jacobian coordinates. For a given +// (x, y) position on the curve, the Jacobian coordinates are (x1, y1, z1) +// where x = x1/z1² and y = y1/z1³. The greatest speedups come when the whole +// calculation can be performed within the transform (as in ScalarMult and +// ScalarBaseMult). But even for Add and Double, it's faster to apply and +// reverse the transform than to operate in affine coordinates. + +import ( + "big" + "io" + "os" + "sync" +) + +// A Curve represents a short-form Weierstrass curve with a=-3. +// See http://www.hyperelliptic.org/EFD/g1p/auto-shortw.html +type Curve struct { + P *big.Int // the order of the underlying field + B *big.Int // the constant of the curve equation + Gx, Gy *big.Int // (x,y) of the base point + BitSize int // the size of the underlying field +} + +// IsOnCurve returns true if the given (x,y) lies on the curve. +func (curve *Curve) IsOnCurve(x, y *big.Int) bool { + // y² = x³ - 3x + b + y2 := new(big.Int).Mul(y, y) + y2.Mod(y2, curve.P) + + x3 := new(big.Int).Mul(x, x) + x3.Mul(x3, x) + + threeX := new(big.Int).Lsh(x, 1) + threeX.Add(threeX, x) + + x3.Sub(x3, threeX) + x3.Add(x3, curve.B) + x3.Mod(x3, curve.P) + + return x3.Cmp(y2) == 0 +} + +// affineFromJacobian reverses the Jacobian transform. See the comment at the +// top of the file. +func (curve *Curve) affineFromJacobian(x, y, z *big.Int) (xOut, yOut *big.Int) { + zinv := new(big.Int).ModInverse(z, curve.P) + zinvsq := new(big.Int).Mul(zinv, zinv) + + xOut = new(big.Int).Mul(x, zinvsq) + xOut.Mod(xOut, curve.P) + zinvsq.Mul(zinvsq, zinv) + yOut = new(big.Int).Mul(y, zinvsq) + yOut.Mod(yOut, curve.P) + return +} + +// Add returns the sum of (x1,y1) and (x2,y2) +func (curve *Curve) Add(x1, y1, x2, y2 *big.Int) (*big.Int, *big.Int) { + z := new(big.Int).SetInt64(1) + return curve.affineFromJacobian(curve.addJacobian(x1, y1, z, x2, y2, z)) +} + +// addJacobian takes two points in Jacobian coordinates, (x1, y1, z1) and +// (x2, y2, z2) and returns their sum, also in Jacobian form. +func (curve *Curve) addJacobian(x1, y1, z1, x2, y2, z2 *big.Int) (*big.Int, *big.Int, *big.Int) { + // See http://hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-3.html#addition-add-2007-bl + z1z1 := new(big.Int).Mul(z1, z1) + z1z1.Mod(z1z1, curve.P) + z2z2 := new(big.Int).Mul(z2, z2) + z2z2.Mod(z2z2, curve.P) + + u1 := new(big.Int).Mul(x1, z2z2) + u1.Mod(u1, curve.P) + u2 := new(big.Int).Mul(x2, z1z1) + u2.Mod(u2, curve.P) + h := new(big.Int).Sub(u2, u1) + if h.Sign() == -1 { + h.Add(h, curve.P) + } + i := new(big.Int).Lsh(h, 1) + i.Mul(i, i) + j := new(big.Int).Mul(h, i) + + s1 := new(big.Int).Mul(y1, z2) + s1.Mul(s1, z2z2) + s1.Mod(s1, curve.P) + s2 := new(big.Int).Mul(y2, z1) + s2.Mul(s2, z1z1) + s2.Mod(s2, curve.P) + r := new(big.Int).Sub(s2, s1) + if r.Sign() == -1 { + r.Add(r, curve.P) + } + r.Lsh(r, 1) + v := new(big.Int).Mul(u1, i) + + x3 := new(big.Int).Set(r) + x3.Mul(x3, x3) + x3.Sub(x3, j) + x3.Sub(x3, v) + x3.Sub(x3, v) + x3.Mod(x3, curve.P) + + y3 := new(big.Int).Set(r) + v.Sub(v, x3) + y3.Mul(y3, v) + s1.Mul(s1, j) + s1.Lsh(s1, 1) + y3.Sub(y3, s1) + y3.Mod(y3, curve.P) + + z3 := new(big.Int).Add(z1, z2) + z3.Mul(z3, z3) + z3.Sub(z3, z1z1) + if z3.Sign() == -1 { + z3.Add(z3, curve.P) + } + z3.Sub(z3, z2z2) + if z3.Sign() == -1 { + z3.Add(z3, curve.P) + } + z3.Mul(z3, h) + z3.Mod(z3, curve.P) + + return x3, y3, z3 +} + +// Double returns 2*(x,y) +func (curve *Curve) Double(x1, y1 *big.Int) (*big.Int, *big.Int) { + z1 := new(big.Int).SetInt64(1) + return curve.affineFromJacobian(curve.doubleJacobian(x1, y1, z1)) +} + +// doubleJacobian takes a point in Jacobian coordinates, (x, y, z), and +// returns its double, also in Jacobian form. +func (curve *Curve) doubleJacobian(x, y, z *big.Int) (*big.Int, *big.Int, *big.Int) { + // See http://hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-3.html#doubling-dbl-2001-b + delta := new(big.Int).Mul(z, z) + delta.Mod(delta, curve.P) + gamma := new(big.Int).Mul(y, y) + gamma.Mod(gamma, curve.P) + alpha := new(big.Int).Sub(x, delta) + if alpha.Sign() == -1 { + alpha.Add(alpha, curve.P) + } + alpha2 := new(big.Int).Add(x, delta) + alpha.Mul(alpha, alpha2) + alpha2.Set(alpha) + alpha.Lsh(alpha, 1) + alpha.Add(alpha, alpha2) + + beta := alpha2.Mul(x, gamma) + + x3 := new(big.Int).Mul(alpha, alpha) + beta8 := new(big.Int).Lsh(beta, 3) + x3.Sub(x3, beta8) + for x3.Sign() == -1 { + x3.Add(x3, curve.P) + } + x3.Mod(x3, curve.P) + + z3 := new(big.Int).Add(y, z) + z3.Mul(z3, z3) + z3.Sub(z3, gamma) + if z3.Sign() == -1 { + z3.Add(z3, curve.P) + } + z3.Sub(z3, delta) + if z3.Sign() == -1 { + z3.Add(z3, curve.P) + } + z3.Mod(z3, curve.P) + + beta.Lsh(beta, 2) + beta.Sub(beta, x3) + if beta.Sign() == -1 { + beta.Add(beta, curve.P) + } + y3 := alpha.Mul(alpha, beta) + + gamma.Mul(gamma, gamma) + gamma.Lsh(gamma, 3) + gamma.Mod(gamma, curve.P) + + y3.Sub(y3, gamma) + if y3.Sign() == -1 { + y3.Add(y3, curve.P) + } + y3.Mod(y3, curve.P) + + return x3, y3, z3 +} + +// ScalarMult returns k*(Bx,By) where k is a number in big-endian form. +func (curve *Curve) ScalarMult(Bx, By *big.Int, k []byte) (*big.Int, *big.Int) { + // We have a slight problem in that the identity of the group (the + // point at infinity) cannot be represented in (x, y) form on a finite + // machine. Thus the standard add/double algorithm has to be tweaked + // slightly: our initial state is not the identity, but x, and we + // ignore the first true bit in |k|. If we don't find any true bits in + // |k|, then we return nil, nil, because we cannot return the identity + // element. + + Bz := new(big.Int).SetInt64(1) + x := Bx + y := By + z := Bz + + seenFirstTrue := false + for _, byte := range k { + for bitNum := 0; bitNum < 8; bitNum++ { + if seenFirstTrue { + x, y, z = curve.doubleJacobian(x, y, z) + } + if byte&0x80 == 0x80 { + if !seenFirstTrue { + seenFirstTrue = true + } else { + x, y, z = curve.addJacobian(Bx, By, Bz, x, y, z) + } + } + byte <<= 1 + } + } + + if !seenFirstTrue { + return nil, nil + } + + return curve.affineFromJacobian(x, y, z) +} + +// ScalarBaseMult returns k*G, where G is the base point of the group and k is +// an integer in big-endian form. +func (curve *Curve) ScalarBaseMult(k []byte) (*big.Int, *big.Int) { + return curve.ScalarMult(curve.Gx, curve.Gy, k) +} + +var mask = []byte{0xff, 0x1, 0x3, 0x7, 0xf, 0x1f, 0x3f, 0x7f} + +// GenerateKey returns a public/private key pair. The private key is generated +// using the given reader, which must return random data. +func (curve *Curve) GenerateKey(rand io.Reader) (priv []byte, x, y *big.Int, err os.Error) { + byteLen := (curve.BitSize + 7) >> 3 + priv = make([]byte, byteLen) + + for x == nil { + _, err = io.ReadFull(rand, priv) + if err != nil { + return + } + // We have to mask off any excess bits in the case that the size of the + // underlying field is not a whole number of bytes. + priv[0] &= mask[curve.BitSize%8] + // This is because, in tests, rand will return all zeros and we don't + // want to get the point at infinity and loop forever. + priv[1] ^= 0x42 + x, y = curve.ScalarBaseMult(priv) + } + return +} + +// Marshal converts a point into the form specified in section 4.3.6 of ANSI +// X9.62. +func (curve *Curve) Marshal(x, y *big.Int) []byte { + byteLen := (curve.BitSize + 7) >> 3 + + ret := make([]byte, 1+2*byteLen) + ret[0] = 4 // uncompressed point + + xBytes := x.Bytes() + copy(ret[1+byteLen-len(xBytes):], xBytes) + yBytes := y.Bytes() + copy(ret[1+2*byteLen-len(yBytes):], yBytes) + return ret +} + +// Unmarshal converts a point, serialised by Marshal, into an x, y pair. On +// error, x = nil. +func (curve *Curve) Unmarshal(data []byte) (x, y *big.Int) { + byteLen := (curve.BitSize + 7) >> 3 + if len(data) != 1+2*byteLen { + return + } + if data[0] != 4 { // uncompressed form + return + } + x = new(big.Int).SetBytes(data[1 : 1+byteLen]) + y = new(big.Int).SetBytes(data[1+byteLen:]) + return +} + +var initonce sync.Once +var p224 *Curve +var p256 *Curve +var p384 *Curve +var p521 *Curve + +func initAll() { + initP224() + initP256() + initP384() + initP521() +} + +func initP224() { + // See FIPS 186-3, section D.2.2 + p224 = new(Curve) + p224.P, _ = new(big.Int).SetString("26959946667150639794667015087019630673557916260026308143510066298881", 10) + p224.B, _ = new(big.Int).SetString("b4050a850c04b3abf54132565044b0b7d7bfd8ba270b39432355ffb4", 16) + p224.Gx, _ = new(big.Int).SetString("b70e0cbd6bb4bf7f321390b94a03c1d356c21122343280d6115c1d21", 16) + p224.Gy, _ = new(big.Int).SetString("bd376388b5f723fb4c22dfe6cd4375a05a07476444d5819985007e34", 16) + p224.BitSize = 224 +} + +func initP256() { + // See FIPS 186-3, section D.2.3 + p256 = new(Curve) + p256.P, _ = new(big.Int).SetString("115792089210356248762697446949407573530086143415290314195533631308867097853951", 10) + p256.B, _ = new(big.Int).SetString("5ac635d8aa3a93e7b3ebbd55769886bc651d06b0cc53b0f63bce3c3e27d2604b", 16) + p256.Gx, _ = new(big.Int).SetString("6b17d1f2e12c4247f8bce6e563a440f277037d812deb33a0f4a13945d898c296", 16) + p256.Gy, _ = new(big.Int).SetString("4fe342e2fe1a7f9b8ee7eb4a7c0f9e162bce33576b315ececbb6406837bf51f5", 16) + p256.BitSize = 256 +} + +func initP384() { + // See FIPS 186-3, section D.2.4 + p384 = new(Curve) + p384.P, _ = new(big.Int).SetString("39402006196394479212279040100143613805079739270465446667948293404245721771496870329047266088258938001861606973112319", 10) + p384.B, _ = new(big.Int).SetString("b3312fa7e23ee7e4988e056be3f82d19181d9c6efe8141120314088f5013875ac656398d8a2ed19d2a85c8edd3ec2aef", 16) + p384.Gx, _ = new(big.Int).SetString("aa87ca22be8b05378eb1c71ef320ad746e1d3b628ba79b9859f741e082542a385502f25dbf55296c3a545e3872760ab7", 16) + p384.Gy, _ = new(big.Int).SetString("3617de4a96262c6f5d9e98bf9292dc29f8f41dbd289a147ce9da3113b5f0b8c00a60b1ce1d7e819d7a431d7c90ea0e5f", 16) + p384.BitSize = 384 +} + +func initP521() { + // See FIPS 186-3, section D.2.5 + p521 = new(Curve) + p521.P, _ = new(big.Int).SetString("6864797660130609714981900799081393217269435300143305409394463459185543183397656052122559640661454554977296311391480858037121987999716643812574028291115057151", 10) + p521.B, _ = new(big.Int).SetString("051953eb9618e1c9a1f929a21a0b68540eea2da725b99b315f3b8b489918ef109e156193951ec7e937b1652c0bd3bb1bf073573df883d2c34f1ef451fd46b503f00", 16) + p521.Gx, _ = new(big.Int).SetString("c6858e06b70404e9cd9e3ecb662395b4429c648139053fb521f828af606b4d3dbaa14b5e77efe75928fe1dc127a2ffa8de3348b3c1856a429bf97e7e31c2e5bd66", 16) + p521.Gy, _ = new(big.Int).SetString("11839296a789a3bc0045c8a5fb42c7d1bd998f54449579b446817afbd17273e662c97ee72995ef42640c550b9013fad0761353c7086a272c24088be94769fd16650", 16) + p521.BitSize = 521 +} + +// P224 returns a Curve which implements P-224 (see FIPS 186-3, section D.2.2) +func P224() *Curve { + initonce.Do(initAll) + return p224 +} + +// P256 returns a Curve which implements P-256 (see FIPS 186-3, section D.2.3) +func P256() *Curve { + initonce.Do(initAll) + return p256 +} + +// P384 returns a Curve which implements P-384 (see FIPS 186-3, section D.2.4) +func P384() *Curve { + initonce.Do(initAll) + return p384 +} + +// P256 returns a Curve which implements P-521 (see FIPS 186-3, section D.2.5) +func P521() *Curve { + initonce.Do(initAll) + return p521 +} diff --git a/libgo/go/crypto/elliptic/elliptic_test.go b/libgo/go/crypto/elliptic/elliptic_test.go new file mode 100644 index 000000000..6ae6fb96d --- /dev/null +++ b/libgo/go/crypto/elliptic/elliptic_test.go @@ -0,0 +1,331 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package elliptic + +import ( + "big" + "crypto/rand" + "fmt" + "testing" +) + +func TestOnCurve(t *testing.T) { + p224 := P224() + if !p224.IsOnCurve(p224.Gx, p224.Gy) { + t.Errorf("FAIL") + } +} + +type baseMultTest struct { + k string + x, y string +} + +var p224BaseMultTests = []baseMultTest{ + { + "1", + "b70e0cbd6bb4bf7f321390b94a03c1d356c21122343280d6115c1d21", + "bd376388b5f723fb4c22dfe6cd4375a05a07476444d5819985007e34", + }, + { + "2", + "706a46dc76dcb76798e60e6d89474788d16dc18032d268fd1a704fa6", + "1c2b76a7bc25e7702a704fa986892849fca629487acf3709d2e4e8bb", + }, + { + "3", + "df1b1d66a551d0d31eff822558b9d2cc75c2180279fe0d08fd896d04", + "a3f7f03cadd0be444c0aa56830130ddf77d317344e1af3591981a925", + }, + { + "4", + "ae99feebb5d26945b54892092a8aee02912930fa41cd114e40447301", + "482580a0ec5bc47e88bc8c378632cd196cb3fa058a7114eb03054c9", + }, + { + "5", + "31c49ae75bce7807cdff22055d94ee9021fedbb5ab51c57526f011aa", + "27e8bff1745635ec5ba0c9f1c2ede15414c6507d29ffe37e790a079b", + }, + { + "6", + "1f2483f82572251fca975fea40db821df8ad82a3c002ee6c57112408", + "89faf0ccb750d99b553c574fad7ecfb0438586eb3952af5b4b153c7e", + }, + { + "7", + "db2f6be630e246a5cf7d99b85194b123d487e2d466b94b24a03c3e28", + "f3a30085497f2f611ee2517b163ef8c53b715d18bb4e4808d02b963", + }, + { + "8", + "858e6f9cc6c12c31f5df124aa77767b05c8bc021bd683d2b55571550", + "46dcd3ea5c43898c5c5fc4fdac7db39c2f02ebee4e3541d1e78047a", + }, + { + "9", + "2fdcccfee720a77ef6cb3bfbb447f9383117e3daa4a07e36ed15f78d", + "371732e4f41bf4f7883035e6a79fcedc0e196eb07b48171697517463", + }, + { + "10", + "aea9e17a306517eb89152aa7096d2c381ec813c51aa880e7bee2c0fd", + "39bb30eab337e0a521b6cba1abe4b2b3a3e524c14a3fe3eb116b655f", + }, + { + "11", + "ef53b6294aca431f0f3c22dc82eb9050324f1d88d377e716448e507c", + "20b510004092e96636cfb7e32efded8265c266dfb754fa6d6491a6da", + }, + { + "12", + "6e31ee1dc137f81b056752e4deab1443a481033e9b4c93a3044f4f7a", + "207dddf0385bfdeab6e9acda8da06b3bbef224a93ab1e9e036109d13", + }, + { + "13", + "34e8e17a430e43289793c383fac9774247b40e9ebd3366981fcfaeca", + "252819f71c7fb7fbcb159be337d37d3336d7feb963724fdfb0ecb767", + }, + { + "14", + "a53640c83dc208603ded83e4ecf758f24c357d7cf48088b2ce01e9fa", + "d5814cd724199c4a5b974a43685fbf5b8bac69459c9469bc8f23ccaf", + }, + { + "15", + "baa4d8635511a7d288aebeedd12ce529ff102c91f97f867e21916bf9", + "979a5f4759f80f4fb4ec2e34f5566d595680a11735e7b61046127989", + }, + { + "16", + "b6ec4fe1777382404ef679997ba8d1cc5cd8e85349259f590c4c66d", + "3399d464345906b11b00e363ef429221f2ec720d2f665d7dead5b482", + }, + { + "17", + "b8357c3a6ceef288310e17b8bfeff9200846ca8c1942497c484403bc", + "ff149efa6606a6bd20ef7d1b06bd92f6904639dce5174db6cc554a26", + }, + { + "18", + "c9ff61b040874c0568479216824a15eab1a838a797d189746226e4cc", + "ea98d60e5ffc9b8fcf999fab1df7e7ef7084f20ddb61bb045a6ce002", + }, + { + "19", + "a1e81c04f30ce201c7c9ace785ed44cc33b455a022f2acdbc6cae83c", + "dcf1f6c3db09c70acc25391d492fe25b4a180babd6cea356c04719cd", + }, + { + "20", + "fcc7f2b45df1cd5a3c0c0731ca47a8af75cfb0347e8354eefe782455", + "d5d7110274cba7cdee90e1a8b0d394c376a5573db6be0bf2747f530", + }, + { + "112233445566778899", + "61f077c6f62ed802dad7c2f38f5c67f2cc453601e61bd076bb46179e", + "2272f9e9f5933e70388ee652513443b5e289dd135dcc0d0299b225e4", + }, + { + "112233445566778899112233445566778899", + "29895f0af496bfc62b6ef8d8a65c88c613949b03668aab4f0429e35", + "3ea6e53f9a841f2019ec24bde1a75677aa9b5902e61081c01064de93", + }, + { + "6950511619965839450988900688150712778015737983940691968051900319680", + "ab689930bcae4a4aa5f5cb085e823e8ae30fd365eb1da4aba9cf0379", + "3345a121bbd233548af0d210654eb40bab788a03666419be6fbd34e7", + }, + { + "13479972933410060327035789020509431695094902435494295338570602119423", + "bdb6a8817c1f89da1c2f3dd8e97feb4494f2ed302a4ce2bc7f5f4025", + "4c7020d57c00411889462d77a5438bb4e97d177700bf7243a07f1680", + }, + { + "13479971751745682581351455311314208093898607229429740618390390702079", + "d58b61aa41c32dd5eba462647dba75c5d67c83606c0af2bd928446a9", + "d24ba6a837be0460dd107ae77725696d211446c5609b4595976b16bd", + }, + { + "13479972931865328106486971546324465392952975980343228160962702868479", + "dc9fa77978a005510980e929a1485f63716df695d7a0c18bb518df03", + "ede2b016f2ddffc2a8c015b134928275ce09e5661b7ab14ce0d1d403", + }, + { + "11795773708834916026404142434151065506931607341523388140225443265536", + "499d8b2829cfb879c901f7d85d357045edab55028824d0f05ba279ba", + "bf929537b06e4015919639d94f57838fa33fc3d952598dcdbb44d638", + }, + { + "784254593043826236572847595991346435467177662189391577090", + "8246c999137186632c5f9eddf3b1b0e1764c5e8bd0e0d8a554b9cb77", + "e80ed8660bc1cb17ac7d845be40a7a022d3306f116ae9f81fea65947", + }, + { + "13479767645505654746623887797783387853576174193480695826442858012671", + "6670c20afcceaea672c97f75e2e9dd5c8460e54bb38538ebb4bd30eb", + "f280d8008d07a4caf54271f993527d46ff3ff46fd1190a3f1faa4f74", + }, + { + "205688069665150753842126177372015544874550518966168735589597183", + "eca934247425cfd949b795cb5ce1eff401550386e28d1a4c5a8eb", + "d4c01040dba19628931bc8855370317c722cbd9ca6156985f1c2e9ce", + }, + { + "13479966930919337728895168462090683249159702977113823384618282123295", + "ef353bf5c73cd551b96d596fbc9a67f16d61dd9fe56af19de1fba9cd", + "21771b9cdce3e8430c09b3838be70b48c21e15bc09ee1f2d7945b91f", + }, + { + "50210731791415612487756441341851895584393717453129007497216", + "4036052a3091eb481046ad3289c95d3ac905ca0023de2c03ecd451cf", + "d768165a38a2b96f812586a9d59d4136035d9c853a5bf2e1c86a4993", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368041", + "fcc7f2b45df1cd5a3c0c0731ca47a8af75cfb0347e8354eefe782455", + "f2a28eefd8b345832116f1e574f2c6b2c895aa8c24941f40d8b80ad1", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368042", + "a1e81c04f30ce201c7c9ace785ed44cc33b455a022f2acdbc6cae83c", + "230e093c24f638f533dac6e2b6d01da3b5e7f45429315ca93fb8e634", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368043", + "c9ff61b040874c0568479216824a15eab1a838a797d189746226e4cc", + "156729f1a003647030666054e208180f8f7b0df2249e44fba5931fff", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368044", + "b8357c3a6ceef288310e17b8bfeff9200846ca8c1942497c484403bc", + "eb610599f95942df1082e4f9426d086fb9c6231ae8b24933aab5db", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368045", + "b6ec4fe1777382404ef679997ba8d1cc5cd8e85349259f590c4c66d", + "cc662b9bcba6f94ee4ff1c9c10bd6ddd0d138df2d099a282152a4b7f", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368046", + "baa4d8635511a7d288aebeedd12ce529ff102c91f97f867e21916bf9", + "6865a0b8a607f0b04b13d1cb0aa992a5a97f5ee8ca1849efb9ed8678", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368047", + "a53640c83dc208603ded83e4ecf758f24c357d7cf48088b2ce01e9fa", + "2a7eb328dbe663b5a468b5bc97a040a3745396ba636b964370dc3352", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368048", + "34e8e17a430e43289793c383fac9774247b40e9ebd3366981fcfaeca", + "dad7e608e380480434ea641cc82c82cbc92801469c8db0204f13489a", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368049", + "6e31ee1dc137f81b056752e4deab1443a481033e9b4c93a3044f4f7a", + "df82220fc7a4021549165325725f94c3410ddb56c54e161fc9ef62ee", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368050", + "ef53b6294aca431f0f3c22dc82eb9050324f1d88d377e716448e507c", + "df4aefffbf6d1699c930481cd102127c9a3d992048ab05929b6e5927", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368051", + "aea9e17a306517eb89152aa7096d2c381ec813c51aa880e7bee2c0fd", + "c644cf154cc81f5ade49345e541b4d4b5c1adb3eb5c01c14ee949aa2", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368052", + "2fdcccfee720a77ef6cb3bfbb447f9383117e3daa4a07e36ed15f78d", + "c8e8cd1b0be40b0877cfca1958603122f1e6914f84b7e8e968ae8b9e", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368053", + "858e6f9cc6c12c31f5df124aa77767b05c8bc021bd683d2b55571550", + "fb9232c15a3bc7673a3a03b0253824c53d0fd1411b1cabe2e187fb87", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368054", + "db2f6be630e246a5cf7d99b85194b123d487e2d466b94b24a03c3e28", + "f0c5cff7ab680d09ee11dae84e9c1072ac48ea2e744b1b7f72fd469e", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368055", + "1f2483f82572251fca975fea40db821df8ad82a3c002ee6c57112408", + "76050f3348af2664aac3a8b05281304ebc7a7914c6ad50a4b4eac383", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368056", + "31c49ae75bce7807cdff22055d94ee9021fedbb5ab51c57526f011aa", + "d817400e8ba9ca13a45f360e3d121eaaeb39af82d6001c8186f5f866", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368057", + "ae99feebb5d26945b54892092a8aee02912930fa41cd114e40447301", + "fb7da7f5f13a43b81774373c879cd32d6934c05fa758eeb14fcfab38", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368058", + "df1b1d66a551d0d31eff822558b9d2cc75c2180279fe0d08fd896d04", + "5c080fc3522f41bbb3f55a97cfecf21f882ce8cbb1e50ca6e67e56dc", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368059", + "706a46dc76dcb76798e60e6d89474788d16dc18032d268fd1a704fa6", + "e3d4895843da188fd58fb0567976d7b50359d6b78530c8f62d1b1746", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368060", + "b70e0cbd6bb4bf7f321390b94a03c1d356c21122343280d6115c1d21", + "42c89c774a08dc04b3dd201932bc8a5ea5f8b89bbb2a7e667aff81cd", + }, +} + +func TestBaseMult(t *testing.T) { + p224 := P224() + for i, e := range p224BaseMultTests { + k, ok := new(big.Int).SetString(e.k, 10) + if !ok { + t.Errorf("%d: bad value for k: %s", i, e.k) + } + x, y := p224.ScalarBaseMult(k.Bytes()) + if fmt.Sprintf("%x", x) != e.x || fmt.Sprintf("%x", y) != e.y { + t.Errorf("%d: bad output for k=%s: got (%x, %s), want (%s, %s)", i, e.k, x, y, e.x, e.y) + } + } +} + +func BenchmarkBaseMult(b *testing.B) { + b.ResetTimer() + p224 := P224() + e := p224BaseMultTests[25] + k, _ := new(big.Int).SetString(e.k, 10) + b.StartTimer() + for i := 0; i < b.N; i++ { + p224.ScalarBaseMult(k.Bytes()) + } +} + +func TestMarshal(t *testing.T) { + p224 := P224() + _, x, y, err := p224.GenerateKey(rand.Reader) + if err != nil { + t.Error(err) + return + } + serialised := p224.Marshal(x, y) + xx, yy := p224.Unmarshal(serialised) + if xx == nil { + t.Error("failed to unmarshal") + return + } + if xx.Cmp(x) != 0 || yy.Cmp(y) != 0 { + t.Error("unmarshal returned different values") + return + } +} diff --git a/libgo/go/crypto/hmac/hmac.go b/libgo/go/crypto/hmac/hmac.go new file mode 100644 index 000000000..298fb2c06 --- /dev/null +++ b/libgo/go/crypto/hmac/hmac.go @@ -0,0 +1,100 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The hmac package implements the Keyed-Hash Message Authentication Code (HMAC) +// as defined in U.S. Federal Information Processing Standards Publication 198. +// An HMAC is a cryptographic hash that uses a key to sign a message. +// The receiver verifies the hash by recomputing it using the same key. +package hmac + +import ( + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "hash" + "os" +) + +// FIPS 198: +// http://csrc.nist.gov/publications/fips/fips198/fips-198a.pdf + +// key is zero padded to 64 bytes +// ipad = 0x36 byte repeated to 64 bytes +// opad = 0x5c byte repeated to 64 bytes +// hmac = H([key ^ opad] H([key ^ ipad] text)) + +const ( + // NOTE(rsc): This constant is actually the + // underlying hash function's block size. + // HMAC is only conventionally used with + // MD5 and SHA1, and both use 64-byte blocks. + // The hash.Hash interface doesn't provide a + // way to find out the block size. + padSize = 64 +) + +type hmac struct { + size int + key, tmp []byte + outer, inner hash.Hash +} + +func (h *hmac) tmpPad(xor byte) { + for i, k := range h.key { + h.tmp[i] = xor ^ k + } + for i := len(h.key); i < padSize; i++ { + h.tmp[i] = xor + } +} + +func (h *hmac) Sum() []byte { + sum := h.inner.Sum() + h.tmpPad(0x5c) + for i, b := range sum { + h.tmp[padSize+i] = b + } + h.outer.Reset() + h.outer.Write(h.tmp) + return h.outer.Sum() +} + +func (h *hmac) Write(p []byte) (n int, err os.Error) { + return h.inner.Write(p) +} + +func (h *hmac) Size() int { return h.size } + +func (h *hmac) Reset() { + h.inner.Reset() + h.tmpPad(0x36) + h.inner.Write(h.tmp[0:padSize]) +} + +// New returns a new HMAC hash using the given hash generator and key. +func New(h func() hash.Hash, key []byte) hash.Hash { + hm := new(hmac) + hm.outer = h() + hm.inner = h() + hm.size = hm.inner.Size() + hm.tmp = make([]byte, padSize+hm.size) + if len(key) > padSize { + // If key is too big, hash it. + hm.outer.Write(key) + key = hm.outer.Sum() + } + hm.key = make([]byte, len(key)) + copy(hm.key, key) + hm.Reset() + return hm +} + +// NewMD5 returns a new HMAC-MD5 hash using the given key. +func NewMD5(key []byte) hash.Hash { return New(md5.New, key) } + +// NewSHA1 returns a new HMAC-SHA1 hash using the given key. +func NewSHA1(key []byte) hash.Hash { return New(sha1.New, key) } + +// NewSHA256 returns a new HMAC-SHA256 hash using the given key. +func NewSHA256(key []byte) hash.Hash { return New(sha256.New, key) } diff --git a/libgo/go/crypto/hmac/hmac_test.go b/libgo/go/crypto/hmac/hmac_test.go new file mode 100644 index 000000000..40adbad04 --- /dev/null +++ b/libgo/go/crypto/hmac/hmac_test.go @@ -0,0 +1,205 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package hmac + +import ( + "hash" + "fmt" + "testing" +) + +type hmacTest struct { + hash func([]byte) hash.Hash + key []byte + in []byte + out string +} + +var hmacTests = []hmacTest{ + // Tests from US FIPS 198 + // http://csrc.nist.gov/publications/fips/fips198/fips-198a.pdf + { + NewSHA1, + []byte{ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, + }, + []byte("Sample #1"), + "4f4ca3d5d68ba7cc0a1208c9c61e9c5da0403c0a", + }, + { + NewSHA1, + []byte{ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, + 0x40, 0x41, 0x42, 0x43, + }, + []byte("Sample #2"), + "0922d3405faa3d194f82a45830737d5cc6c75d24", + }, + { + NewSHA1, + []byte{ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, + 0xb0, 0xb1, 0xb2, 0xb3, + }, + []byte("Sample #3"), + "bcf41eab8bb2d802f3d05caf7cb092ecf8d1a3aa", + }, + + // Test from Plan 9. + { + NewMD5, + []byte("Jefe"), + []byte("what do ya want for nothing?"), + "750c783e6ab0b503eaa86e310a5db738", + }, + + // Tests from RFC 4231 + { + NewSHA256, + []byte{ + 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, + 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, + 0x0b, 0x0b, 0x0b, 0x0b, + }, + []byte("Hi There"), + "b0344c61d8db38535ca8afceaf0bf12b881dc200c9833da726e9376c2e32cff7", + }, + { + NewSHA256, + []byte("Jefe"), + []byte("what do ya want for nothing?"), + "5bdcc146bf60754e6a042426089575c75a003f089d2739839dec58b964ec3843", + }, + { + NewSHA256, + []byte{ + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, + }, + []byte{ + 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, + 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, + 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, + 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, + 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, + 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, + 0xdd, 0xdd, + }, + "773ea91e36800e46854db8ebd09181a72959098b3ef8c122d9635514ced565fe", + }, + { + NewSHA256, + []byte{ + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, + 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, + 0x19, + }, + []byte{ + 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, + 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, + 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, + 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, + 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, + 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, + 0xcd, 0xcd, + }, + "82558a389a443c0ea4cc819899f2083a85f0faa3e578f8077a2e3ff46729665b", + }, + { + NewSHA256, + []byte{ + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, + }, + []byte("Test Using Larger Than Block-Size Key - Hash Key First"), + "60e431591ee0b67f0d8a26aacbf5b77f8e0bc6213728c5140546040f0ee37f54", + }, + { + NewSHA256, + []byte{ + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, + }, + []byte("This is a test using a larger than block-size key " + + "and a larger than block-size data. The key needs to " + + "be hashed before being used by the HMAC algorithm."), + "9b09ffa71b942fcb27635fbcd5b0e944bfdc63644f0713938a7f51535c3a35e2", + }, +} + +func TestHMAC(t *testing.T) { + for i, tt := range hmacTests { + h := tt.hash(tt.key) + for j := 0; j < 2; j++ { + n, err := h.Write(tt.in) + if n != len(tt.in) || err != nil { + t.Errorf("test %d.%d: Write(%d) = %d, %v", i, j, len(tt.in), n, err) + continue + } + + // Repetive Sum() calls should return the same value + for k := 0; k < 2; k++ { + sum := fmt.Sprintf("%x", h.Sum()) + if sum != tt.out { + t.Errorf("test %d.%d.%d: have %s want %s\n", i, j, k, sum, tt.out) + } + } + + // Second iteration: make sure reset works. + h.Reset() + } + } +} diff --git a/libgo/go/crypto/md4/md4.go b/libgo/go/crypto/md4/md4.go new file mode 100644 index 000000000..e13c986e6 --- /dev/null +++ b/libgo/go/crypto/md4/md4.go @@ -0,0 +1,112 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This package implements the MD4 hash algorithm as defined in RFC 1320. +package md4 + +import ( + "hash" + "os" +) + +// The size of an MD4 checksum in bytes. +const Size = 16 + +const ( + _Chunk = 64 + _Init0 = 0x67452301 + _Init1 = 0xEFCDAB89 + _Init2 = 0x98BADCFE + _Init3 = 0x10325476 +) + +// digest represents the partial evaluation of a checksum. +type digest struct { + s [4]uint32 + x [_Chunk]byte + nx int + len uint64 +} + +func (d *digest) Reset() { + d.s[0] = _Init0 + d.s[1] = _Init1 + d.s[2] = _Init2 + d.s[3] = _Init3 + d.nx = 0 + d.len = 0 +} + +// New returns a new hash.Hash computing the MD4 checksum. +func New() hash.Hash { + d := new(digest) + d.Reset() + return d +} + +func (d *digest) Size() int { return Size } + +func (d *digest) Write(p []byte) (nn int, err os.Error) { + nn = len(p) + d.len += uint64(nn) + if d.nx > 0 { + n := len(p) + if n > _Chunk-d.nx { + n = _Chunk - d.nx + } + for i := 0; i < n; i++ { + d.x[d.nx+i] = p[i] + } + d.nx += n + if d.nx == _Chunk { + _Block(d, d.x[0:]) + d.nx = 0 + } + p = p[n:] + } + n := _Block(d, p) + p = p[n:] + if len(p) > 0 { + d.nx = copy(d.x[:], p) + } + return +} + +func (d0 *digest) Sum() []byte { + // Make a copy of d0, so that caller can keep writing and summing. + d := new(digest) + *d = *d0 + + // Padding. Add a 1 bit and 0 bits until 56 bytes mod 64. + len := d.len + var tmp [64]byte + tmp[0] = 0x80 + if len%64 < 56 { + d.Write(tmp[0 : 56-len%64]) + } else { + d.Write(tmp[0 : 64+56-len%64]) + } + + // Length in bits. + len <<= 3 + for i := uint(0); i < 8; i++ { + tmp[i] = byte(len >> (8 * i)) + } + d.Write(tmp[0:8]) + + if d.nx != 0 { + panic("d.nx != 0") + } + + p := make([]byte, 16) + j := 0 + for _, s := range d.s { + p[j+0] = byte(s >> 0) + p[j+1] = byte(s >> 8) + p[j+2] = byte(s >> 16) + p[j+3] = byte(s >> 24) + j += 4 + } + return p +} diff --git a/libgo/go/crypto/md4/md4_test.go b/libgo/go/crypto/md4/md4_test.go new file mode 100644 index 000000000..721bd4cbc --- /dev/null +++ b/libgo/go/crypto/md4/md4_test.go @@ -0,0 +1,71 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package md4 + +import ( + "fmt" + "io" + "testing" +) + +type md4Test struct { + out string + in string +} + +var golden = []md4Test{ + {"31d6cfe0d16ae931b73c59d7e0c089c0", ""}, + {"bde52cb31de33e46245e05fbdbd6fb24", "a"}, + {"ec388dd78999dfc7cf4632465693b6bf", "ab"}, + {"a448017aaf21d8525fc10ae87aa6729d", "abc"}, + {"41decd8f579255c5200f86a4bb3ba740", "abcd"}, + {"9803f4a34e8eb14f96adba49064a0c41", "abcde"}, + {"804e7f1c2586e50b49ac65db5b645131", "abcdef"}, + {"752f4adfe53d1da0241b5bc216d098fc", "abcdefg"}, + {"ad9daf8d49d81988590a6f0e745d15dd", "abcdefgh"}, + {"1e4e28b05464316b56402b3815ed2dfd", "abcdefghi"}, + {"dc959c6f5d6f9e04e4380777cc964b3d", "abcdefghij"}, + {"1b5701e265778898ef7de5623bbe7cc0", "Discard medicine more than two years old."}, + {"d7f087e090fe7ad4a01cb59dacc9a572", "He who has a shady past knows that nice guys finish last."}, + {"a6f8fd6df617c72837592fc3570595c9", "I wouldn't marry him with a ten foot pole."}, + {"c92a84a9526da8abc240c05d6b1a1ce0", "Free! Free!/A trip/to Mars/for 900/empty jars/Burma Shave"}, + {"f6013160c4dcb00847069fee3bb09803", "The days of the digital watch are numbered. -Tom Stoppard"}, + {"2c3bb64f50b9107ed57640fe94bec09f", "Nepal premier won't resign."}, + {"45b7d8a32c7806f2f7f897332774d6e4", "For every action there is an equal and opposite government program."}, + {"b5b4f9026b175c62d7654bdc3a1cd438", "His money is twice tainted: 'taint yours and 'taint mine."}, + {"caf44e80f2c20ce19b5ba1cab766e7bd", "There is no reason for any individual to have a computer in their home. -Ken Olsen, 1977"}, + {"191fae6707f496aa54a6bce9f2ecf74d", "It's a tiny change to the code and not completely disgusting. - Bob Manchek"}, + {"9ddc753e7a4ccee6081cd1b45b23a834", "size: a.out: bad magic"}, + {"8d050f55b1cadb9323474564be08a521", "The major problem is with sendmail. -Mark Horton"}, + {"ad6e2587f74c3e3cc19146f6127fa2e3", "Give me a rock, paper and scissors and I will move the world. CCFestoon"}, + {"1d616d60a5fabe85589c3f1566ca7fca", "If the enemy is within range, then so are you."}, + {"aec3326a4f496a2ced65a1963f84577f", "It's well we cannot hear the screams/That we create in others' dreams."}, + {"77b4fd762d6b9245e61c50bf6ebf118b", "You remind me of a TV show, but that's all right: I watch it anyway."}, + {"e8f48c726bae5e516f6ddb1a4fe62438", "C is as portable as Stonehedge!!"}, + {"a3a84366e7219e887423b01f9be7166e", "Even if I could be Shakespeare, I think I should still choose to be Faraday. - A. Huxley"}, + {"a6b7aa35157e984ef5d9b7f32e5fbb52", "The fugacity of a constituent in a mixture of gases at a given temperature is proportional to its mole fraction. Lewis-Randall Rule"}, + {"75661f0545955f8f9abeeb17845f3fd6", "How can you write a big system without C++? -Paul Glick"}, +} + +func TestGolden(t *testing.T) { + for i := 0; i < len(golden); i++ { + g := golden[i] + c := New() + for j := 0; j < 3; j++ { + if j < 2 { + io.WriteString(c, g.in) + } else { + io.WriteString(c, g.in[0:len(g.in)/2]) + c.Sum() + io.WriteString(c, g.in[len(g.in)/2:]) + } + s := fmt.Sprintf("%x", c.Sum()) + if s != g.out { + t.Fatalf("md4[%d](%s) = %s want %s", j, g.in, s, g.out) + } + c.Reset() + } + } +} diff --git a/libgo/go/crypto/md4/md4block.go b/libgo/go/crypto/md4/md4block.go new file mode 100644 index 000000000..3fed475f3 --- /dev/null +++ b/libgo/go/crypto/md4/md4block.go @@ -0,0 +1,89 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// MD4 block step. +// In its own file so that a faster assembly or C version +// can be substituted easily. + +package md4 + +var shift1 = []uint{3, 7, 11, 19} +var shift2 = []uint{3, 5, 9, 13} +var shift3 = []uint{3, 9, 11, 15} + +var xIndex2 = []uint{0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15} +var xIndex3 = []uint{0, 8, 4, 12, 2, 10, 6, 14, 1, 9, 5, 13, 3, 11, 7, 15} + +func _Block(dig *digest, p []byte) int { + a := dig.s[0] + b := dig.s[1] + c := dig.s[2] + d := dig.s[3] + n := 0 + var X [16]uint32 + for len(p) >= _Chunk { + aa, bb, cc, dd := a, b, c, d + + j := 0 + for i := 0; i < 16; i++ { + X[i] = uint32(p[j]) | uint32(p[j+1])<<8 | uint32(p[j+2])<<16 | uint32(p[j+3])<<24 + j += 4 + } + + // If this needs to be made faster in the future, + // the usual trick is to unroll each of these + // loops by a factor of 4; that lets you replace + // the shift[] lookups with constants and, + // with suitable variable renaming in each + // unrolled body, delete the a, b, c, d = d, a, b, c + // (or you can let the optimizer do the renaming). + // + // The index variables are uint so that % by a power + // of two can be optimized easily by a compiler. + + // Round 1. + for i := uint(0); i < 16; i++ { + x := i + s := shift1[i%4] + f := ((c ^ d) & b) ^ d + a += f + X[x] + a = a<>(32-s) + a, b, c, d = d, a, b, c + } + + // Round 2. + for i := uint(0); i < 16; i++ { + x := xIndex2[i] + s := shift2[i%4] + g := (b & c) | (b & d) | (c & d) + a += g + X[x] + 0x5a827999 + a = a<>(32-s) + a, b, c, d = d, a, b, c + } + + // Round 3. + for i := uint(0); i < 16; i++ { + x := xIndex3[i] + s := shift3[i%4] + h := b ^ c ^ d + a += h + X[x] + 0x6ed9eba1 + a = a<>(32-s) + a, b, c, d = d, a, b, c + } + + a += aa + b += bb + c += cc + d += dd + + p = p[_Chunk:] + n += _Chunk + } + + dig.s[0] = a + dig.s[1] = b + dig.s[2] = c + dig.s[3] = d + return n +} diff --git a/libgo/go/crypto/md5/md5.go b/libgo/go/crypto/md5/md5.go new file mode 100644 index 000000000..54fddb63b --- /dev/null +++ b/libgo/go/crypto/md5/md5.go @@ -0,0 +1,112 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This package implements the MD5 hash algorithm as defined in RFC 1321. +package md5 + +import ( + "hash" + "os" +) + +// The size of an MD5 checksum in bytes. +const Size = 16 + +const ( + _Chunk = 64 + _Init0 = 0x67452301 + _Init1 = 0xEFCDAB89 + _Init2 = 0x98BADCFE + _Init3 = 0x10325476 +) + +// digest represents the partial evaluation of a checksum. +type digest struct { + s [4]uint32 + x [_Chunk]byte + nx int + len uint64 +} + +func (d *digest) Reset() { + d.s[0] = _Init0 + d.s[1] = _Init1 + d.s[2] = _Init2 + d.s[3] = _Init3 + d.nx = 0 + d.len = 0 +} + +// New returns a new hash.Hash computing the MD5 checksum. +func New() hash.Hash { + d := new(digest) + d.Reset() + return d +} + +func (d *digest) Size() int { return Size } + +func (d *digest) Write(p []byte) (nn int, err os.Error) { + nn = len(p) + d.len += uint64(nn) + if d.nx > 0 { + n := len(p) + if n > _Chunk-d.nx { + n = _Chunk - d.nx + } + for i := 0; i < n; i++ { + d.x[d.nx+i] = p[i] + } + d.nx += n + if d.nx == _Chunk { + _Block(d, d.x[0:]) + d.nx = 0 + } + p = p[n:] + } + n := _Block(d, p) + p = p[n:] + if len(p) > 0 { + d.nx = copy(d.x[:], p) + } + return +} + +func (d0 *digest) Sum() []byte { + // Make a copy of d0 so that caller can keep writing and summing. + d := new(digest) + *d = *d0 + + // Padding. Add a 1 bit and 0 bits until 56 bytes mod 64. + len := d.len + var tmp [64]byte + tmp[0] = 0x80 + if len%64 < 56 { + d.Write(tmp[0 : 56-len%64]) + } else { + d.Write(tmp[0 : 64+56-len%64]) + } + + // Length in bits. + len <<= 3 + for i := uint(0); i < 8; i++ { + tmp[i] = byte(len >> (8 * i)) + } + d.Write(tmp[0:8]) + + if d.nx != 0 { + panic("d.nx != 0") + } + + p := make([]byte, 16) + j := 0 + for _, s := range d.s { + p[j+0] = byte(s >> 0) + p[j+1] = byte(s >> 8) + p[j+2] = byte(s >> 16) + p[j+3] = byte(s >> 24) + j += 4 + } + return p +} diff --git a/libgo/go/crypto/md5/md5_test.go b/libgo/go/crypto/md5/md5_test.go new file mode 100644 index 000000000..857002b70 --- /dev/null +++ b/libgo/go/crypto/md5/md5_test.go @@ -0,0 +1,71 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package md5 + +import ( + "fmt" + "io" + "testing" +) + +type md5Test struct { + out string + in string +} + +var golden = []md5Test{ + {"d41d8cd98f00b204e9800998ecf8427e", ""}, + {"0cc175b9c0f1b6a831c399e269772661", "a"}, + {"187ef4436122d1cc2f40dc2b92f0eba0", "ab"}, + {"900150983cd24fb0d6963f7d28e17f72", "abc"}, + {"e2fc714c4727ee9395f324cd2e7f331f", "abcd"}, + {"ab56b4d92b40713acc5af89985d4b786", "abcde"}, + {"e80b5017098950fc58aad83c8c14978e", "abcdef"}, + {"7ac66c0f148de9519b8bd264312c4d64", "abcdefg"}, + {"e8dc4081b13434b45189a720b77b6818", "abcdefgh"}, + {"8aa99b1f439ff71293e95357bac6fd94", "abcdefghi"}, + {"a925576942e94b2ef57a066101b48876", "abcdefghij"}, + {"d747fc1719c7eacb84058196cfe56d57", "Discard medicine more than two years old."}, + {"bff2dcb37ef3a44ba43ab144768ca837", "He who has a shady past knows that nice guys finish last."}, + {"0441015ecb54a7342d017ed1bcfdbea5", "I wouldn't marry him with a ten foot pole."}, + {"9e3cac8e9e9757a60c3ea391130d3689", "Free! Free!/A trip/to Mars/for 900/empty jars/Burma Shave"}, + {"a0f04459b031f916a59a35cc482dc039", "The days of the digital watch are numbered. -Tom Stoppard"}, + {"e7a48e0fe884faf31475d2a04b1362cc", "Nepal premier won't resign."}, + {"637d2fe925c07c113800509964fb0e06", "For every action there is an equal and opposite government program."}, + {"834a8d18d5c6562119cf4c7f5086cb71", "His money is twice tainted: 'taint yours and 'taint mine."}, + {"de3a4d2fd6c73ec2db2abad23b444281", "There is no reason for any individual to have a computer in their home. -Ken Olsen, 1977"}, + {"acf203f997e2cf74ea3aff86985aefaf", "It's a tiny change to the code and not completely disgusting. - Bob Manchek"}, + {"e1c1384cb4d2221dfdd7c795a4222c9a", "size: a.out: bad magic"}, + {"c90f3ddecc54f34228c063d7525bf644", "The major problem is with sendmail. -Mark Horton"}, + {"cdf7ab6c1fd49bd9933c43f3ea5af185", "Give me a rock, paper and scissors and I will move the world. CCFestoon"}, + {"83bc85234942fc883c063cbd7f0ad5d0", "If the enemy is within range, then so are you."}, + {"277cbe255686b48dd7e8f389394d9299", "It's well we cannot hear the screams/That we create in others' dreams."}, + {"fd3fb0a7ffb8af16603f3d3af98f8e1f", "You remind me of a TV show, but that's all right: I watch it anyway."}, + {"469b13a78ebf297ecda64d4723655154", "C is as portable as Stonehedge!!"}, + {"63eb3a2f466410104731c4b037600110", "Even if I could be Shakespeare, I think I should still choose to be Faraday. - A. Huxley"}, + {"72c2ed7592debca1c90fc0100f931a2f", "The fugacity of a constituent in a mixture of gases at a given temperature is proportional to its mole fraction. Lewis-Randall Rule"}, + {"132f7619d33b523b1d9e5bd8e0928355", "How can you write a big system without C++? -Paul Glick"}, +} + +func TestGolden(t *testing.T) { + for i := 0; i < len(golden); i++ { + g := golden[i] + c := New() + for j := 0; j < 3; j++ { + if j < 2 { + io.WriteString(c, g.in) + } else { + io.WriteString(c, g.in[0:len(g.in)/2]) + c.Sum() + io.WriteString(c, g.in[len(g.in)/2:]) + } + s := fmt.Sprintf("%x", c.Sum()) + if s != g.out { + t.Fatalf("md5[%d](%s) = %s want %s", j, g.in, s, g.out) + } + c.Reset() + } + } +} diff --git a/libgo/go/crypto/md5/md5block.go b/libgo/go/crypto/md5/md5block.go new file mode 100644 index 000000000..a887e2e05 --- /dev/null +++ b/libgo/go/crypto/md5/md5block.go @@ -0,0 +1,172 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// MD5 block step. +// In its own file so that a faster assembly or C version +// can be substituted easily. + +package md5 + +// table[i] = int((1<<32) * abs(sin(i+1 radians))). +var table = []uint32{ + // round 1 + 0xd76aa478, + 0xe8c7b756, + 0x242070db, + 0xc1bdceee, + 0xf57c0faf, + 0x4787c62a, + 0xa8304613, + 0xfd469501, + 0x698098d8, + 0x8b44f7af, + 0xffff5bb1, + 0x895cd7be, + 0x6b901122, + 0xfd987193, + 0xa679438e, + 0x49b40821, + + // round 2 + 0xf61e2562, + 0xc040b340, + 0x265e5a51, + 0xe9b6c7aa, + 0xd62f105d, + 0x2441453, + 0xd8a1e681, + 0xe7d3fbc8, + 0x21e1cde6, + 0xc33707d6, + 0xf4d50d87, + 0x455a14ed, + 0xa9e3e905, + 0xfcefa3f8, + 0x676f02d9, + 0x8d2a4c8a, + + // round3 + 0xfffa3942, + 0x8771f681, + 0x6d9d6122, + 0xfde5380c, + 0xa4beea44, + 0x4bdecfa9, + 0xf6bb4b60, + 0xbebfbc70, + 0x289b7ec6, + 0xeaa127fa, + 0xd4ef3085, + 0x4881d05, + 0xd9d4d039, + 0xe6db99e5, + 0x1fa27cf8, + 0xc4ac5665, + + // round 4 + 0xf4292244, + 0x432aff97, + 0xab9423a7, + 0xfc93a039, + 0x655b59c3, + 0x8f0ccc92, + 0xffeff47d, + 0x85845dd1, + 0x6fa87e4f, + 0xfe2ce6e0, + 0xa3014314, + 0x4e0811a1, + 0xf7537e82, + 0xbd3af235, + 0x2ad7d2bb, + 0xeb86d391, +} + +var shift1 = []uint{7, 12, 17, 22} +var shift2 = []uint{5, 9, 14, 20} +var shift3 = []uint{4, 11, 16, 23} +var shift4 = []uint{6, 10, 15, 21} + +func _Block(dig *digest, p []byte) int { + a := dig.s[0] + b := dig.s[1] + c := dig.s[2] + d := dig.s[3] + n := 0 + var X [16]uint32 + for len(p) >= _Chunk { + aa, bb, cc, dd := a, b, c, d + + j := 0 + for i := 0; i < 16; i++ { + X[i] = uint32(p[j]) | uint32(p[j+1])<<8 | uint32(p[j+2])<<16 | uint32(p[j+3])<<24 + j += 4 + } + + // If this needs to be made faster in the future, + // the usual trick is to unroll each of these + // loops by a factor of 4; that lets you replace + // the shift[] lookups with constants and, + // with suitable variable renaming in each + // unrolled body, delete the a, b, c, d = d, a, b, c + // (or you can let the optimizer do the renaming). + // + // The index variables are uint so that % by a power + // of two can be optimized easily by a compiler. + + // Round 1. + for i := uint(0); i < 16; i++ { + x := i + s := shift1[i%4] + f := ((c ^ d) & b) ^ d + a += f + X[x] + table[i] + a = a<>(32-s) + b + a, b, c, d = d, a, b, c + } + + // Round 2. + for i := uint(0); i < 16; i++ { + x := (1 + 5*i) % 16 + s := shift2[i%4] + g := ((b ^ c) & d) ^ c + a += g + X[x] + table[i+16] + a = a<>(32-s) + b + a, b, c, d = d, a, b, c + } + + // Round 3. + for i := uint(0); i < 16; i++ { + x := (5 + 3*i) % 16 + s := shift3[i%4] + h := b ^ c ^ d + a += h + X[x] + table[i+32] + a = a<>(32-s) + b + a, b, c, d = d, a, b, c + } + + // Round 4. + for i := uint(0); i < 16; i++ { + x := (7 * i) % 16 + s := shift4[i%4] + j := c ^ (b | ^d) + a += j + X[x] + table[i+48] + a = a<>(32-s) + b + a, b, c, d = d, a, b, c + } + + a += aa + b += bb + c += cc + d += dd + + p = p[_Chunk:] + n += _Chunk + } + + dig.s[0] = a + dig.s[1] = b + dig.s[2] = c + dig.s[3] = d + return n +} diff --git a/libgo/go/crypto/ocsp/ocsp.go b/libgo/go/crypto/ocsp/ocsp.go new file mode 100644 index 000000000..f3fa3bc83 --- /dev/null +++ b/libgo/go/crypto/ocsp/ocsp.go @@ -0,0 +1,203 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This package parses OCSP responses as specified in RFC 2560. OCSP responses +// are signed messages attesting to the validity of a certificate for a small +// period of time. This is used to manage revocation for X.509 certificates. +package ocsp + +import ( + "asn1" + "crypto/rsa" + "crypto/sha1" + "crypto/x509" + "os" + "time" +) + +var idPKIXOCSPBasic = asn1.ObjectIdentifier([]int{1, 3, 6, 1, 5, 5, 7, 48, 1, 1}) +var idSHA1WithRSA = asn1.ObjectIdentifier([]int{1, 2, 840, 113549, 1, 1, 5}) + +// These are internal structures that reflect the ASN.1 structure of an OCSP +// response. See RFC 2560, section 4.2. + +const ( + ocspSuccess = 0 + ocspMalformed = 1 + ocspInternalError = 2 + ocspTryLater = 3 + ocspSigRequired = 4 + ocspUnauthorized = 5 +) + +type rdnSequence []relativeDistinguishedNameSET + +type relativeDistinguishedNameSET []attributeTypeAndValue + +type attributeTypeAndValue struct { + Type asn1.ObjectIdentifier + Value interface{} +} + +type algorithmIdentifier struct { + Algorithm asn1.ObjectIdentifier +} + +type certID struct { + HashAlgorithm algorithmIdentifier + NameHash []byte + IssuerKeyHash []byte + SerialNumber asn1.RawValue +} + +type responseASN1 struct { + Status asn1.Enumerated + Response responseBytes "explicit,tag:0" +} + +type responseBytes struct { + ResponseType asn1.ObjectIdentifier + Response []byte +} + +type basicResponse struct { + TBSResponseData responseData + SignatureAlgorithm algorithmIdentifier + Signature asn1.BitString + Certificates []asn1.RawValue "explicit,tag:0,optional" +} + +type responseData struct { + Raw asn1.RawContent + Version int "optional,default:1,explicit,tag:0" + RequestorName rdnSequence "optional,explicit,tag:1" + KeyHash []byte "optional,explicit,tag:2" + ProducedAt *time.Time + Responses []singleResponse +} + +type singleResponse struct { + CertID certID + Good asn1.Flag "explicit,tag:0,optional" + Revoked revokedInfo "explicit,tag:1,optional" + Unknown asn1.Flag "explicit,tag:2,optional" + ThisUpdate *time.Time + NextUpdate *time.Time "explicit,tag:0,optional" +} + +type revokedInfo struct { + RevocationTime *time.Time + Reason int "explicit,tag:0,optional" +} + +// This is the exposed reflection of the internal OCSP structures. + +const ( + // Good means that the certificate is valid. + Good = iota + // Revoked means that the certificate has been deliberately revoked. + Revoked = iota + // Unknown means that the OCSP responder doesn't know about the certificate. + Unknown = iota + // ServerFailed means that the OCSP responder failed to process the request. + ServerFailed = iota +) + +// Response represents an OCSP response. See RFC 2560. +type Response struct { + // Status is one of {Good, Revoked, Unknown, ServerFailed} + Status int + SerialNumber []byte + ProducedAt, ThisUpdate, NextUpdate, RevokedAt *time.Time + RevocationReason int + Certificate *x509.Certificate +} + +// ParseError results from an invalid OCSP response. +type ParseError string + +func (p ParseError) String() string { + return string(p) +} + +// ParseResponse parses an OCSP response in DER form. It only supports +// responses for a single certificate and only those using RSA signatures. +// Non-RSA responses will result in an x509.UnsupportedAlgorithmError. +// Signature errors or parse failures will result in a ParseError. +func ParseResponse(bytes []byte) (*Response, os.Error) { + var resp responseASN1 + rest, err := asn1.Unmarshal(bytes, &resp) + if err != nil { + return nil, err + } + if len(rest) > 0 { + return nil, ParseError("trailing data in OCSP response") + } + + ret := new(Response) + if resp.Status != ocspSuccess { + ret.Status = ServerFailed + return ret, nil + } + + if !resp.Response.ResponseType.Equal(idPKIXOCSPBasic) { + return nil, ParseError("bad OCSP response type") + } + + var basicResp basicResponse + rest, err = asn1.Unmarshal(resp.Response.Response, &basicResp) + if err != nil { + return nil, err + } + + if len(basicResp.Certificates) != 1 { + return nil, ParseError("OCSP response contains bad number of certificates") + } + + if len(basicResp.TBSResponseData.Responses) != 1 { + return nil, ParseError("OCSP response contains bad number of responses") + } + + ret.Certificate, err = x509.ParseCertificate(basicResp.Certificates[0].FullBytes) + if err != nil { + return nil, err + } + + if ret.Certificate.PublicKeyAlgorithm != x509.RSA || !basicResp.SignatureAlgorithm.Algorithm.Equal(idSHA1WithRSA) { + return nil, x509.UnsupportedAlgorithmError{} + } + + h := sha1.New() + hashType := rsa.HashSHA1 + + pub := ret.Certificate.PublicKey.(*rsa.PublicKey) + h.Write(basicResp.TBSResponseData.Raw) + digest := h.Sum() + signature := basicResp.Signature.RightAlign() + + if rsa.VerifyPKCS1v15(pub, hashType, digest, signature) != nil { + return nil, ParseError("bad OCSP signature") + } + + r := basicResp.TBSResponseData.Responses[0] + + ret.SerialNumber = r.CertID.SerialNumber.Bytes + + switch { + case bool(r.Good): + ret.Status = Good + case bool(r.Unknown): + ret.Status = Unknown + default: + ret.Status = Revoked + ret.RevokedAt = r.Revoked.RevocationTime + ret.RevocationReason = r.Revoked.Reason + } + + ret.ProducedAt = basicResp.TBSResponseData.ProducedAt + ret.ThisUpdate = r.ThisUpdate + ret.NextUpdate = r.NextUpdate + + return ret, nil +} diff --git a/libgo/go/crypto/ocsp/ocsp_test.go b/libgo/go/crypto/ocsp/ocsp_test.go new file mode 100644 index 000000000..f9889790f --- /dev/null +++ b/libgo/go/crypto/ocsp/ocsp_test.go @@ -0,0 +1,97 @@ +package ocsp + +import ( + "bytes" + "encoding/hex" + "reflect" + "testing" + "time" +) + +func TestOCSPDecode(t *testing.T) { + responseBytes, _ := hex.DecodeString(ocspResponseHex) + resp, err := ParseResponse(responseBytes) + if err != nil { + t.Error(err) + } + + expected := Response{Status: 0, SerialNumber: []byte{0x1, 0xd0, 0xfa}, RevocationReason: 0, ThisUpdate: &time.Time{Year: 2010, Month: 7, Day: 7, Hour: 15, Minute: 1, Second: 5, Weekday: 0, ZoneOffset: 0, Zone: "UTC"}, NextUpdate: &time.Time{Year: 2010, Month: 7, Day: 7, Hour: 18, Minute: 35, Second: 17, Weekday: 0, ZoneOffset: 0, Zone: "UTC"}} + + if !reflect.DeepEqual(resp.ThisUpdate, resp.ThisUpdate) { + t.Errorf("resp.ThisUpdate: got %d, want %d", resp.ThisUpdate, expected.ThisUpdate) + } + + if !reflect.DeepEqual(resp.NextUpdate, resp.NextUpdate) { + t.Errorf("resp.NextUpdate: got %d, want %d", resp.NextUpdate, expected.NextUpdate) + } + + if resp.Status != expected.Status { + t.Errorf("resp.Status: got %d, want %d", resp.Status, expected.Status) + } + + if !bytes.Equal(resp.SerialNumber, expected.SerialNumber) { + t.Errorf("resp.SerialNumber: got %x, want %x", resp.SerialNumber, expected.SerialNumber) + } + + if resp.RevocationReason != expected.RevocationReason { + t.Errorf("resp.RevocationReason: got %d, want %d", resp.RevocationReason, expected.RevocationReason) + } +} + +// This OCSP response was taken from Thawte's public OCSP responder. +// To recreate: +// $ openssl s_client -tls1 -showcerts -servername www.google.com -connect www.google.com:443 +// Copy and paste the first certificate into /tmp/cert.crt and the second into +// /tmp/intermediate.crt +// $ openssl ocsp -issuer /tmp/intermediate.crt -cert /tmp/cert.crt -url http://ocsp.thawte.com -resp_text -respout /tmp/ocsp.der +// Then hex encode the result: +// $ python -c 'print file("/tmp/ocsp.der", "r").read().encode("hex")' + +const ocspResponseHex = "308206bc0a0100a08206b5308206b106092b0601050507300101048206a23082069e3081" + + "c9a14e304c310b300906035504061302494c31163014060355040a130d5374617274436f" + + "6d204c74642e312530230603550403131c5374617274436f6d20436c6173732031204f43" + + "5350205369676e6572180f32303130303730373137333531375a30663064303c30090605" + + "2b0e03021a050004146568874f40750f016a3475625e1f5c93e5a26d580414eb4234d098" + + "b0ab9ff41b6b08f7cc642eef0e2c45020301d0fa8000180f323031303037303731353031" + + "30355aa011180f32303130303730373138333531375a300d06092a864886f70d01010505" + + "000382010100ab557ff070d1d7cebbb5f0ec91a15c3fed22eb2e1b8244f1b84545f013a4" + + "fb46214c5e3fbfbebb8a56acc2b9db19f68fd3c3201046b3824d5ba689f99864328710cb" + + "467195eb37d84f539e49f859316b32964dc3e47e36814ce94d6c56dd02733b1d0802f7ff" + + "4eebdbbd2927dcf580f16cbc290f91e81b53cb365e7223f1d6e20a88ea064104875e0145" + + "672b20fc14829d51ca122f5f5d77d3ad6c83889c55c7dc43680ba2fe3cef8b05dbcabdc0" + + "d3e09aaf9725597f8c858c2fa38c0d6aed2e6318194420dd1a1137445d13e1c97ab47896" + + "17a4e08925f46f867b72e3a4dc1f08cb870b2b0717f7207faa0ac512e628a029aba7457a" + + "e63dcf3281e2162d9349a08204ba308204b6308204b23082039aa003020102020101300d" + + "06092a864886f70d010105050030818c310b300906035504061302494c31163014060355" + + "040a130d5374617274436f6d204c74642e312b3029060355040b13225365637572652044" + + "69676974616c204365727469666963617465205369676e696e6731383036060355040313" + + "2f5374617274436f6d20436c6173732031205072696d61727920496e7465726d65646961" + + "746520536572766572204341301e170d3037313032353030323330365a170d3132313032" + + "333030323330365a304c310b300906035504061302494c31163014060355040a130d5374" + + "617274436f6d204c74642e312530230603550403131c5374617274436f6d20436c617373" + + "2031204f435350205369676e657230820122300d06092a864886f70d0101010500038201" + + "0f003082010a0282010100b9561b4c45318717178084e96e178df2255e18ed8d8ecc7c2b" + + "7b51a6c1c2e6bf0aa3603066f132fe10ae97b50e99fa24b83fc53dd2777496387d14e1c3" + + "a9b6a4933e2ac12413d085570a95b8147414a0bc007c7bcf222446ef7f1a156d7ea1c577" + + "fc5f0facdfd42eb0f5974990cb2f5cefebceef4d1bdc7ae5c1075c5a99a93171f2b0845b" + + "4ff0864e973fcfe32f9d7511ff87a3e943410c90a4493a306b6944359340a9ca96f02b66" + + "ce67f028df2980a6aaee8d5d5d452b8b0eb93f923cc1e23fcccbdbe7ffcb114d08fa7a6a" + + "3c404f825d1a0e715935cf623a8c7b59670014ed0622f6089a9447a7a19010f7fe58f841" + + "29a2765ea367824d1c3bb2fda308530203010001a382015c30820158300c0603551d1301" + + "01ff04023000300b0603551d0f0404030203a8301e0603551d250417301506082b060105" + + "0507030906092b0601050507300105301d0603551d0e0416041445e0a36695414c5dd449" + + "bc00e33cdcdbd2343e173081a80603551d230481a030819d8014eb4234d098b0ab9ff41b" + + "6b08f7cc642eef0e2c45a18181a47f307d310b300906035504061302494c311630140603" + + "55040a130d5374617274436f6d204c74642e312b3029060355040b132253656375726520" + + "4469676974616c204365727469666963617465205369676e696e67312930270603550403" + + "13205374617274436f6d2043657274696669636174696f6e20417574686f726974798201" + + "0a30230603551d12041c301a8618687474703a2f2f7777772e737461727473736c2e636f" + + "6d2f302c06096086480186f842010d041f161d5374617274436f6d205265766f63617469" + + "6f6e20417574686f72697479300d06092a864886f70d01010505000382010100182d2215" + + "8f0fc0291324fa8574c49bb8ff2835085adcbf7b7fc4191c397ab6951328253fffe1e5ec" + + "2a7da0d50fca1a404e6968481366939e666c0a6209073eca57973e2fefa9ed1718e8176f" + + "1d85527ff522c08db702e3b2b180f1cbff05d98128252cf0f450f7dd2772f4188047f19d" + + "c85317366f94bc52d60f453a550af58e308aaab00ced33040b62bf37f5b1ab2a4f7f0f80" + + "f763bf4d707bc8841d7ad9385ee2a4244469260b6f2bf085977af9074796048ecc2f9d48" + + "a1d24ce16e41a9941568fec5b42771e118f16c106a54ccc339a4b02166445a167902e75e" + + "6d8620b0825dcd18a069b90fd851d10fa8effd409deec02860d26d8d833f304b10669b42" diff --git a/libgo/go/crypto/openpgp/armor/armor.go b/libgo/go/crypto/openpgp/armor/armor.go new file mode 100644 index 000000000..97080f6c6 --- /dev/null +++ b/libgo/go/crypto/openpgp/armor/armor.go @@ -0,0 +1,220 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This package implements OpenPGP ASCII Armor, see RFC 4880. OpenPGP Armor is +// very similar to PEM except that it has an additional CRC checksum. +package armor + +import ( + "bytes" + "crypto/openpgp/error" + "encoding/base64" + "encoding/line" + "io" + "os" +) + +// A Block represents an OpenPGP armored structure. +// +// The encoded form is: +// -----BEGIN Type----- +// Headers +// +// base64-encoded Bytes +// '=' base64 encoded checksum +// -----END Type----- +// where Headers is a possibly empty sequence of Key: Value lines. +// +// Since the armored data can be very large, this package presents a streaming +// interface. +type Block struct { + Type string // The type, taken from the preamble (i.e. "PGP SIGNATURE"). + Header map[string]string // Optional headers. + Body io.Reader // A Reader from which the contents can be read + lReader lineReader + oReader openpgpReader +} + +var ArmorCorrupt os.Error = error.StructuralError("armor invalid") + +const crc24Init = 0xb704ce +const crc24Poly = 0x1864cfb +const crc24Mask = 0xffffff + +// crc24 calculates the OpenPGP checksum as specified in RFC 4880, section 6.1 +func crc24(crc uint32, d []byte) uint32 { + for _, b := range d { + crc ^= uint32(b) << 16 + for i := 0; i < 8; i++ { + crc <<= 1 + if crc&0x1000000 != 0 { + crc ^= crc24Poly + } + } + } + return crc +} + +var armorStart = []byte("-----BEGIN ") +var armorEnd = []byte("-----END ") +var armorEndOfLine = []byte("-----") + +// lineReader wraps a line based reader. It watches for the end of an armor +// block and records the expected CRC value. +type lineReader struct { + in *line.Reader + buf []byte + eof bool + crc uint32 +} + +func (l *lineReader) Read(p []byte) (n int, err os.Error) { + if l.eof { + return 0, os.EOF + } + + if len(l.buf) > 0 { + n = copy(p, l.buf) + l.buf = l.buf[n:] + return + } + + line, isPrefix, err := l.in.ReadLine() + if err != nil { + return + } + if isPrefix { + return 0, ArmorCorrupt + } + + if len(line) == 5 && line[0] == '=' { + // This is the checksum line + var expectedBytes [3]byte + var m int + m, err = base64.StdEncoding.Decode(expectedBytes[0:], line[1:]) + if m != 3 || err != nil { + return + } + l.crc = uint32(expectedBytes[0])<<16 | + uint32(expectedBytes[1])<<8 | + uint32(expectedBytes[2]) + + line, _, err = l.in.ReadLine() + if err != nil && err != os.EOF { + return + } + if !bytes.HasPrefix(line, armorEnd) { + return 0, ArmorCorrupt + } + + l.eof = true + return 0, os.EOF + } + + if len(line) != 64 { + return 0, ArmorCorrupt + } + + n = copy(p, line) + bytesToSave := len(line) - n + if bytesToSave > 0 { + if cap(l.buf) < bytesToSave { + l.buf = make([]byte, 0, bytesToSave) + } + l.buf = l.buf[0:bytesToSave] + copy(l.buf, line[n:]) + } + + return +} + +// openpgpReader passes Read calls to the underlying base64 decoder, but keeps +// a running CRC of the resulting data and checks the CRC against the value +// found by the lineReader at EOF. +type openpgpReader struct { + lReader *lineReader + b64Reader io.Reader + currentCRC uint32 +} + +func (r *openpgpReader) Read(p []byte) (n int, err os.Error) { + n, err = r.b64Reader.Read(p) + r.currentCRC = crc24(r.currentCRC, p[:n]) + + if err == os.EOF { + if r.lReader.crc != uint32(r.currentCRC&crc24Mask) { + return 0, ArmorCorrupt + } + } + + return +} + +// Decode reads a PGP armored block from the given Reader. It will ignore +// leading garbage. If it doesn't find a block, it will return nil, os.EOF. The +// given Reader is not usable after calling this function: an arbitary amount +// of data may have been read past the end of the block. +func Decode(in io.Reader) (p *Block, err os.Error) { + r := line.NewReader(in, 100) + var line []byte + ignoreNext := false + +TryNextBlock: + p = nil + + // Skip leading garbage + for { + ignoreThis := ignoreNext + line, ignoreNext, err = r.ReadLine() + if err != nil { + return + } + if ignoreNext || ignoreThis { + continue + } + line = bytes.TrimSpace(line) + if len(line) > len(armorStart)+len(armorEndOfLine) && bytes.HasPrefix(line, armorStart) { + break + } + } + + p = new(Block) + p.Type = string(line[len(armorStart) : len(line)-len(armorEndOfLine)]) + p.Header = make(map[string]string) + nextIsContinuation := false + var lastKey string + + // Read headers + for { + isContinuation := nextIsContinuation + line, nextIsContinuation, err = r.ReadLine() + if err != nil { + p = nil + return + } + if isContinuation { + p.Header[lastKey] += string(line) + continue + } + line = bytes.TrimSpace(line) + if len(line) == 0 { + break + } + + i := bytes.Index(line, []byte(": ")) + if i == -1 { + goto TryNextBlock + } + lastKey = string(line[:i]) + p.Header[lastKey] = string(line[i+2:]) + } + + p.lReader.in = r + p.oReader.currentCRC = crc24Init + p.oReader.lReader = &p.lReader + p.oReader.b64Reader = base64.NewDecoder(base64.StdEncoding, &p.lReader) + p.Body = &p.oReader + + return +} diff --git a/libgo/go/crypto/openpgp/armor/armor_test.go b/libgo/go/crypto/openpgp/armor/armor_test.go new file mode 100644 index 000000000..e4ffd414b --- /dev/null +++ b/libgo/go/crypto/openpgp/armor/armor_test.go @@ -0,0 +1,97 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package armor + +import ( + "bytes" + "hash/adler32" + "io/ioutil" + "testing" +) + +func TestDecodeEncode(t *testing.T) { + buf := bytes.NewBuffer([]byte(armorExample1)) + result, err := Decode(buf) + if err != nil { + t.Error(err) + } + expectedType := "PGP SIGNATURE" + if result.Type != expectedType { + t.Errorf("result.Type: got:%s want:%s", result.Type, expectedType) + } + if len(result.Header) != 1 { + t.Errorf("len(result.Header): got:%d want:1", len(result.Header)) + } + v, ok := result.Header["Version"] + if !ok || v != "GnuPG v1.4.10 (GNU/Linux)" { + t.Errorf("result.Header: got:%#v", result.Header) + } + + contents, err := ioutil.ReadAll(result.Body) + if err != nil { + t.Error(err) + } + + if adler32.Checksum(contents) != 0x789d7f00 { + t.Errorf("contents: got: %x", contents) + } + + buf = bytes.NewBuffer(nil) + w, err := Encode(buf, result.Type, result.Header) + if err != nil { + t.Error(err) + } + _, err = w.Write(contents) + if err != nil { + t.Error(err) + } + w.Close() + + if !bytes.Equal(buf.Bytes(), []byte(armorExample1)) { + t.Errorf("got: %s\nwant: %s", string(buf.Bytes()), armorExample1) + } +} + +func TestLongHeader(t *testing.T) { + buf := bytes.NewBuffer([]byte(armorLongLine)) + result, err := Decode(buf) + if err != nil { + t.Error(err) + return + } + value, ok := result.Header["Version"] + if !ok { + t.Errorf("missing Version header") + } + if value != longValueExpected { + t.Errorf("got: %s want: %s", value, longValueExpected) + } +} + +const armorExample1 = `-----BEGIN PGP SIGNATURE----- +Version: GnuPG v1.4.10 (GNU/Linux) + +iQEcBAABAgAGBQJMtFESAAoJEKsQXJGvOPsVj40H/1WW6jaMXv4BW+1ueDSMDwM8 +kx1fLOXbVM5/Kn5LStZNt1jWWnpxdz7eq3uiqeCQjmqUoRde3YbB2EMnnwRbAhpp +cacnAvy9ZQ78OTxUdNW1mhX5bS6q1MTEJnl+DcyigD70HG/yNNQD7sOPMdYQw0TA +byQBwmLwmTsuZsrYqB68QyLHI+DUugn+kX6Hd2WDB62DKa2suoIUIHQQCd/ofwB3 +WfCYInXQKKOSxu2YOg2Eb4kLNhSMc1i9uKUWAH+sdgJh7NBgdoE4MaNtBFkHXRvv +okWuf3+xA9ksp1npSY/mDvgHijmjvtpRDe6iUeqfCn8N9u9CBg8geANgaG8+QA4= +=wfQG +-----END PGP SIGNATURE-----` + +const armorLongLine = `-----BEGIN PGP SIGNATURE----- +Version: 0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz + +iQEcBAABAgAGBQJMtFESAAoJEKsQXJGvOPsVj40H/1WW6jaMXv4BW+1ueDSMDwM8 +kx1fLOXbVM5/Kn5LStZNt1jWWnpxdz7eq3uiqeCQjmqUoRde3YbB2EMnnwRbAhpp +cacnAvy9ZQ78OTxUdNW1mhX5bS6q1MTEJnl+DcyigD70HG/yNNQD7sOPMdYQw0TA +byQBwmLwmTsuZsrYqB68QyLHI+DUugn+kX6Hd2WDB62DKa2suoIUIHQQCd/ofwB3 +WfCYInXQKKOSxu2YOg2Eb4kLNhSMc1i9uKUWAH+sdgJh7NBgdoE4MaNtBFkHXRvv +okWuf3+xA9ksp1npSY/mDvgHijmjvtpRDe6iUeqfCn8N9u9CBg8geANgaG8+QA4= +=wfQG +-----END PGP SIGNATURE-----` + +const longValueExpected = "0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz" diff --git a/libgo/go/crypto/openpgp/armor/encode.go b/libgo/go/crypto/openpgp/armor/encode.go new file mode 100644 index 000000000..410e73460 --- /dev/null +++ b/libgo/go/crypto/openpgp/armor/encode.go @@ -0,0 +1,162 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package armor + +import ( + "encoding/base64" + "io" + "os" +) + +var armorHeaderSep = []byte(": ") +var blockEnd = []byte("\n=") +var newline = []byte("\n") +var armorEndOfLineOut = []byte("-----\n") + +// writeSlices writes its arguments to the given Writer. +func writeSlices(out io.Writer, slices ...[]byte) (err os.Error) { + for _, s := range slices { + _, err := out.Write(s) + if err != nil { + return + } + } + return +} + +// lineBreaker breaks data across several lines, all of the same byte length +// (except possibly the last). Lines are broken with a single '\n'. +type lineBreaker struct { + lineLength int + line []byte + used int + out io.Writer + haveWritten bool +} + +func newLineBreaker(out io.Writer, lineLength int) *lineBreaker { + return &lineBreaker{ + lineLength: lineLength, + line: make([]byte, lineLength), + used: 0, + out: out, + } +} + +func (l *lineBreaker) Write(b []byte) (n int, err os.Error) { + n = len(b) + + if n == 0 { + return + } + + if l.used == 0 && l.haveWritten { + _, err = l.out.Write([]byte{'\n'}) + if err != nil { + return + } + } + + if l.used+len(b) < l.lineLength { + l.used += copy(l.line[l.used:], b) + return + } + + l.haveWritten = true + _, err = l.out.Write(l.line[0:l.used]) + if err != nil { + return + } + excess := l.lineLength - l.used + l.used = 0 + + _, err = l.out.Write(b[0:excess]) + if err != nil { + return + } + + _, err = l.Write(b[excess:]) + return +} + +func (l *lineBreaker) Close() (err os.Error) { + if l.used > 0 { + _, err = l.out.Write(l.line[0:l.used]) + if err != nil { + return + } + } + + return +} + +// encoding keeps track of a running CRC24 over the data which has been written +// to it and outputs a OpenPGP checksum when closed, followed by an armor +// trailer. +// +// It's built into a stack of io.Writers: +// encoding -> base64 encoder -> lineBreaker -> out +type encoding struct { + out io.Writer + breaker *lineBreaker + b64 io.WriteCloser + crc uint32 + blockType []byte +} + +func (e *encoding) Write(data []byte) (n int, err os.Error) { + e.crc = crc24(e.crc, data) + return e.b64.Write(data) +} + +func (e *encoding) Close() (err os.Error) { + err = e.b64.Close() + if err != nil { + return + } + + var checksumBytes [3]byte + checksumBytes[0] = byte(e.crc >> 16) + checksumBytes[1] = byte(e.crc >> 8) + checksumBytes[2] = byte(e.crc) + + var b64ChecksumBytes [4]byte + base64.StdEncoding.Encode(b64ChecksumBytes[:], checksumBytes[:]) + + return writeSlices(e.out, blockEnd, b64ChecksumBytes[:], newline, armorEnd, e.blockType, armorEndOfLine) +} + +// Encode returns a WriteCloser which will encode the data written to it in +// OpenPGP armor. +func Encode(out io.Writer, blockType string, headers map[string]string) (w io.WriteCloser, err os.Error) { + bType := []byte(blockType) + err = writeSlices(out, armorStart, bType, armorEndOfLineOut) + if err != nil { + return + } + + for k, v := range headers { + err = writeSlices(out, []byte(k), armorHeaderSep, []byte(v), newline) + if err != nil { + return + } + } + + if len(headers) > 0 { + _, err := out.Write(newline) + if err != nil { + return + } + } + + e := &encoding{ + out: out, + breaker: newLineBreaker(out, 64), + crc: crc24Init, + blockType: bType, + } + e.b64 = base64.NewEncoder(base64.StdEncoding, e.breaker) + return e, nil +} diff --git a/libgo/go/crypto/openpgp/error/error.go b/libgo/go/crypto/openpgp/error/error.go new file mode 100644 index 000000000..2d80ce373 --- /dev/null +++ b/libgo/go/crypto/openpgp/error/error.go @@ -0,0 +1,46 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This package contains common error types for the OpenPGP packages. +package error + +// A StructuralError is returned when OpenPGP data is found to be syntactically +// invalid. +type StructuralError string + +func (s StructuralError) String() string { + return "OpenPGP data invalid: " + string(s) +} + +// UnsupportedError indicates that, although the OpenPGP data is valid, it +// makes use of currently unimplemented features. +type UnsupportedError string + +func (s UnsupportedError) String() string { + return "OpenPGP feature unsupported: " + string(s) +} + +// InvalidArgumentError indicates that the caller is in error and passed an +// incorrect value. +type InvalidArgumentError string + +func (i InvalidArgumentError) String() string { + return "OpenPGP argument invalid: " + string(i) +} + +// SignatureError indicates that a syntactically valid signature failed to +// validate. +type SignatureError string + +func (b SignatureError) String() string { + return "OpenPGP signature invalid: " + string(b) +} + +type keyIncorrect int + +func (ki keyIncorrect) String() string { + return "the given key was incorrect" +} + +var KeyIncorrectError = keyIncorrect(0) diff --git a/libgo/go/crypto/openpgp/s2k/s2k.go b/libgo/go/crypto/openpgp/s2k/s2k.go new file mode 100644 index 000000000..f369d7ed4 --- /dev/null +++ b/libgo/go/crypto/openpgp/s2k/s2k.go @@ -0,0 +1,146 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This package implements the various OpenPGP string-to-key transforms as +// specified in RFC 4800 section 3.7.1. +package s2k + +import ( + "crypto/md5" + "crypto/openpgp/error" + "crypto/ripemd160" + "crypto/sha1" + "crypto/sha256" + "crypto/sha512" + "hash" + "io" + "os" +) + +// Simple writes to out the result of computing the Simple S2K function (RFC +// 4880, section 3.7.1.1) using the given hash and input passphrase. +func Simple(out []byte, h hash.Hash, in []byte) { + Salted(out, h, in, nil) +} + +var zero [1]byte + +// Salted writes to out the result of computing the Salted S2K function (RFC +// 4880, section 3.7.1.2) using the given hash, input passphrase and salt. +func Salted(out []byte, h hash.Hash, in []byte, salt []byte) { + done := 0 + + for i := 0; done < len(out); i++ { + h.Reset() + for j := 0; j < i; j++ { + h.Write(zero[:]) + } + h.Write(salt) + h.Write(in) + n := copy(out[done:], h.Sum()) + done += n + } +} + +// Iterated writes to out the result of computing the Iterated and Salted S2K +// function (RFC 4880, section 3.7.1.3) using the given hash, input passphrase, +// salt and iteration count. +func Iterated(out []byte, h hash.Hash, in []byte, salt []byte, count int) { + combined := make([]byte, len(in)+len(salt)) + copy(combined, salt) + copy(combined[len(salt):], in) + + if count < len(combined) { + count = len(combined) + } + + done := 0 + for i := 0; done < len(out); i++ { + h.Reset() + for j := 0; j < i; j++ { + h.Write(zero[:]) + } + written := 0 + for written < count { + if written+len(combined) > count { + todo := count - written + h.Write(combined[:todo]) + written = count + } else { + h.Write(combined) + written += len(combined) + } + } + n := copy(out[done:], h.Sum()) + done += n + } +} + +// Parse reads a binary specification for a string-to-key transformation from r +// and returns a function which performs that transform. +func Parse(r io.Reader) (f func(out, in []byte), err os.Error) { + var buf [9]byte + + _, err = io.ReadFull(r, buf[:2]) + if err != nil { + return + } + + h := hashFuncFromType(buf[1]) + if h == nil { + return nil, error.UnsupportedError("hash for S2K function") + } + + switch buf[0] { + case 1: + f := func(out, in []byte) { + Simple(out, h, in) + } + return f, nil + case 2: + _, err := io.ReadFull(r, buf[:8]) + if err != nil { + return + } + f := func(out, in []byte) { + Salted(out, h, in, buf[:8]) + } + return f, nil + case 3: + _, err := io.ReadFull(r, buf[:9]) + if err != nil { + return + } + count := (16 + int(buf[8]&15)) << (uint32(buf[8]>>4) + 6) + f := func(out, in []byte) { + Iterated(out, h, in, buf[:8], count) + } + return f, nil + } + + return nil, error.UnsupportedError("S2K function") +} + +// hashFuncFromType returns a hash.Hash which corresponds to the given hash +// type byte. See RFC 4880, section 9.4. +func hashFuncFromType(hashType byte) hash.Hash { + switch hashType { + case 1: + return md5.New() + case 2: + return sha1.New() + case 3: + return ripemd160.New() + case 8: + return sha256.New() + case 9: + return sha512.New384() + case 10: + return sha512.New() + case 11: + return sha256.New224() + } + + return nil +} diff --git a/libgo/go/crypto/openpgp/s2k/s2k_test.go b/libgo/go/crypto/openpgp/s2k/s2k_test.go new file mode 100644 index 000000000..814b78627 --- /dev/null +++ b/libgo/go/crypto/openpgp/s2k/s2k_test.go @@ -0,0 +1,94 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package s2k + +import ( + "bytes" + "crypto/sha1" + "encoding/hex" + "testing" +) + +var saltedTests = []struct { + in, out string +}{ + {"hello", "10295ac1"}, + {"world", "ac587a5e"}, + {"foo", "4dda8077"}, + {"bar", "bd8aac6b9ea9cae04eae6a91c6133b58b5d9a61c14f355516ed9370456"}, + {"x", "f1d3f289"}, + {"xxxxxxxxxxxxxxxxxxxxxxx", "e00d7b45"}, +} + +func TestSalted(t *testing.T) { + h := sha1.New() + salt := [4]byte{1, 2, 3, 4} + + for i, test := range saltedTests { + expected, _ := hex.DecodeString(test.out) + out := make([]byte, len(expected)) + Salted(out, h, []byte(test.in), salt[:]) + if !bytes.Equal(expected, out) { + t.Errorf("#%d, got: %x want: %x", i, out, expected) + } + } +} + + +var iteratedTests = []struct { + in, out string +}{ + {"hello", "83126105"}, + {"world", "6fa317f9"}, + {"foo", "8fbc35b9"}, + {"bar", "2af5a99b54f093789fd657f19bd245af7604d0f6ae06f66602a46a08ae"}, + {"x", "5a684dfe"}, + {"xxxxxxxxxxxxxxxxxxxxxxx", "18955174"}, +} + +func TestIterated(t *testing.T) { + h := sha1.New() + salt := [4]byte{4, 3, 2, 1} + + for i, test := range iteratedTests { + expected, _ := hex.DecodeString(test.out) + out := make([]byte, len(expected)) + Iterated(out, h, []byte(test.in), salt[:], 31) + if !bytes.Equal(expected, out) { + t.Errorf("#%d, got: %x want: %x", i, out, expected) + } + } +} + + +var parseTests = []struct { + spec, in, out string +}{ + /* Simple with SHA1 */ + {"0102", "hello", "aaf4c61d"}, + /* Salted with SHA1 */ + {"02020102030405060708", "hello", "f4f7d67e"}, + /* Iterated with SHA1 */ + {"03020102030405060708f1", "hello", "f2a57b7c"}, +} + +func TestParse(t *testing.T) { + for i, test := range parseTests { + spec, _ := hex.DecodeString(test.spec) + buf := bytes.NewBuffer(spec) + f, err := Parse(buf) + if err != nil { + t.Errorf("%d: Parse returned error: %s", i, err) + continue + } + + expected, _ := hex.DecodeString(test.out) + out := make([]byte, len(expected)) + f(out, []byte(test.in)) + if !bytes.Equal(out, expected) { + t.Errorf("%d: output got: %x want: %x", i, out, expected) + } + } +} diff --git a/libgo/go/crypto/rand/rand.go b/libgo/go/crypto/rand/rand.go new file mode 100644 index 000000000..42d9da0ef --- /dev/null +++ b/libgo/go/crypto/rand/rand.go @@ -0,0 +1,21 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package rand implements a cryptographically secure +// pseudorandom number generator. +package rand + +import ( + "io" + "os" +) + +// Reader is a global, shared instance of a cryptographically +// strong pseudo-random generator. +// On Unix-like systems, Reader reads from /dev/urandom. +// On Windows systems, Reader uses the CryptGenRandom API. +var Reader io.Reader + +// Read is a helper function that calls Reader.Read. +func Read(b []byte) (n int, err os.Error) { return Reader.Read(b) } diff --git a/libgo/go/crypto/rand/rand_test.go b/libgo/go/crypto/rand/rand_test.go new file mode 100644 index 000000000..f64ead4ca --- /dev/null +++ b/libgo/go/crypto/rand/rand_test.go @@ -0,0 +1,27 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package rand + +import ( + "bytes" + "compress/flate" + "testing" +) + +func TestRead(t *testing.T) { + b := make([]byte, 4e6) + n, err := Read(b) + if n != len(b) || err != nil { + t.Fatalf("Read(buf) = %d, %s", n, err) + } + + var z bytes.Buffer + f := flate.NewWriter(&z, 5) + f.Write(b) + f.Close() + if z.Len() < len(b)*99/100 { + t.Fatalf("Compressed %d -> %d", len(b), z.Len()) + } +} diff --git a/libgo/go/crypto/rand/rand_unix.go b/libgo/go/crypto/rand/rand_unix.go new file mode 100644 index 000000000..ff16f2554 --- /dev/null +++ b/libgo/go/crypto/rand/rand_unix.go @@ -0,0 +1,125 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Unix cryptographically secure pseudorandom number +// generator. + +package rand + +import ( + "crypto/aes" + "io" + "os" + "sync" + "time" +) + +// Easy implementation: read from /dev/urandom. +// This is sufficient on Linux, OS X, and FreeBSD. + +func init() { Reader = &devReader{name: "/dev/urandom"} } + +// A devReader satisfies reads by reading the file named name. +type devReader struct { + name string + f *os.File + mu sync.Mutex +} + +func (r *devReader) Read(b []byte) (n int, err os.Error) { + r.mu.Lock() + if r.f == nil { + f, err := os.Open(r.name, os.O_RDONLY, 0) + if f == nil { + r.mu.Unlock() + return 0, err + } + r.f = f + } + r.mu.Unlock() + return r.f.Read(b) +} + +// Alternate pseudo-random implementation for use on +// systems without a reliable /dev/urandom. So far we +// haven't needed it. + +// newReader returns a new pseudorandom generator that +// seeds itself by reading from entropy. If entropy == nil, +// the generator seeds itself by reading from the system's +// random number generator, typically /dev/random. +// The Read method on the returned reader always returns +// the full amount asked for, or else it returns an error. +// +// The generator uses the X9.31 algorithm with AES-128, +// reseeding after every 1 MB of generated data. +func newReader(entropy io.Reader) io.Reader { + if entropy == nil { + entropy = &devReader{name: "/dev/random"} + } + return &reader{entropy: entropy} +} + +type reader struct { + mu sync.Mutex + budget int // number of bytes that can be generated + cipher *aes.Cipher + entropy io.Reader + time, seed, dst, key [aes.BlockSize]byte +} + +func (r *reader) Read(b []byte) (n int, err os.Error) { + r.mu.Lock() + defer r.mu.Unlock() + n = len(b) + + for len(b) > 0 { + if r.budget == 0 { + _, err := io.ReadFull(r.entropy, r.seed[0:]) + if err != nil { + return n - len(b), err + } + _, err = io.ReadFull(r.entropy, r.key[0:]) + if err != nil { + return n - len(b), err + } + r.cipher, err = aes.NewCipher(r.key[0:]) + if err != nil { + return n - len(b), err + } + r.budget = 1 << 20 // reseed after generating 1MB + } + r.budget -= aes.BlockSize + + // ANSI X9.31 (== X9.17) algorithm, but using AES in place of 3DES. + // + // single block: + // t = encrypt(time) + // dst = encrypt(t^seed) + // seed = encrypt(t^dst) + ns := time.Nanoseconds() + r.time[0] = byte(ns >> 56) + r.time[1] = byte(ns >> 48) + r.time[2] = byte(ns >> 40) + r.time[3] = byte(ns >> 32) + r.time[4] = byte(ns >> 24) + r.time[5] = byte(ns >> 16) + r.time[6] = byte(ns >> 8) + r.time[7] = byte(ns) + r.cipher.Encrypt(r.time[0:], r.time[0:]) + for i := 0; i < aes.BlockSize; i++ { + r.dst[i] = r.time[i] ^ r.seed[i] + } + r.cipher.Encrypt(r.dst[0:], r.dst[0:]) + for i := 0; i < aes.BlockSize; i++ { + r.seed[i] = r.time[i] ^ r.dst[i] + } + r.cipher.Encrypt(r.seed[0:], r.seed[0:]) + + m := copy(b, r.dst[0:]) + b = b[m:] + } + + return n, nil +} diff --git a/libgo/go/crypto/rand/rand_windows.go b/libgo/go/crypto/rand/rand_windows.go new file mode 100644 index 000000000..4b2b7a26f --- /dev/null +++ b/libgo/go/crypto/rand/rand_windows.go @@ -0,0 +1,43 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Windows cryptographically secure pseudorandom number +// generator. + +package rand + +import ( + "os" + "sync" + "syscall" +) + +// Implemented by using Windows CryptoAPI 2.0. + +func init() { Reader = &rngReader{} } + +// A rngReader satisfies reads by reading from the Windows CryptGenRandom API. +type rngReader struct { + prov uint32 + mu sync.Mutex +} + +func (r *rngReader) Read(b []byte) (n int, err os.Error) { + r.mu.Lock() + if r.prov == 0 { + const provType = syscall.PROV_RSA_FULL + const flags = syscall.CRYPT_VERIFYCONTEXT | syscall.CRYPT_SILENT + ok, errno := syscall.CryptAcquireContext(&r.prov, nil, nil, provType, flags) + if !ok { + r.mu.Unlock() + return 0, os.NewSyscallError("CryptAcquireContext", errno) + } + } + r.mu.Unlock() + ok, errno := syscall.CryptGenRandom(r.prov, uint32(len(b)), &b[0]) + if !ok { + return 0, os.NewSyscallError("CryptGenRandom", errno) + } + return len(b), nil +} diff --git a/libgo/go/crypto/rc4/rc4.go b/libgo/go/crypto/rc4/rc4.go new file mode 100644 index 000000000..65fd195f3 --- /dev/null +++ b/libgo/go/crypto/rc4/rc4.go @@ -0,0 +1,66 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This package implements RC4 encryption, as defined in Bruce Schneier's +// Applied Cryptography. +package rc4 + +// BUG(agl): RC4 is in common use but has design weaknesses that make +// it a poor choice for new protocols. + +import ( + "os" + "strconv" +) + +// A Cipher is an instance of RC4 using a particular key. +type Cipher struct { + s [256]byte + i, j uint8 +} + +type KeySizeError int + +func (k KeySizeError) String() string { + return "crypto/rc4: invalid key size " + strconv.Itoa(int(k)) +} + +// NewCipher creates and returns a new Cipher. The key argument should be the +// RC4 key, at least 1 byte and at most 256 bytes. +func NewCipher(key []byte) (*Cipher, os.Error) { + k := len(key) + if k < 1 || k > 256 { + return nil, KeySizeError(k) + } + var c Cipher + for i := 0; i < 256; i++ { + c.s[i] = uint8(i) + } + var j uint8 = 0 + for i := 0; i < 256; i++ { + j += c.s[i] + key[i%k] + c.s[i], c.s[j] = c.s[j], c.s[i] + } + return &c, nil +} + +// XORKeyStream sets dst to the result of XORing src with the key stream. +// Dst and src may be the same slice but otherwise should not overlap. +func (c *Cipher) XORKeyStream(dst, src []byte) { + for i := range src { + c.i += 1 + c.j += c.s[c.i] + c.s[c.i], c.s[c.j] = c.s[c.j], c.s[c.i] + dst[i] = src[i] ^ c.s[c.s[c.i]+c.s[c.j]] + } +} + +// Reset zeros the key data so that it will no longer appear in the +// process's memory. +func (c *Cipher) Reset() { + for i := range c.s { + c.s[i] = 0 + } + c.i, c.j = 0, 0 +} diff --git a/libgo/go/crypto/rc4/rc4_test.go b/libgo/go/crypto/rc4/rc4_test.go new file mode 100644 index 000000000..6265d9408 --- /dev/null +++ b/libgo/go/crypto/rc4/rc4_test.go @@ -0,0 +1,59 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package rc4 + +import ( + "testing" +) + +type rc4Test struct { + key, keystream []byte +} + +var golden = []rc4Test{ + // Test vectors from the original cypherpunk posting of ARC4: + // http://groups.google.com/group/sci.crypt/msg/10a300c9d21afca0?pli=1 + { + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef}, + []byte{0x74, 0x94, 0xc2, 0xe7, 0x10, 0x4b, 0x08, 0x79}, + }, + { + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xde, 0x18, 0x89, 0x41, 0xa3, 0x37, 0x5d, 0x3a}, + }, + { + []byte{0xef, 0x01, 0x23, 0x45}, + []byte{0xd6, 0xa1, 0x41, 0xa7, 0xec, 0x3c, 0x38, 0xdf, 0xbd, 0x61}, + }, + + // Test vectors from the Wikipedia page: http://en.wikipedia.org/wiki/RC4 + { + []byte{0x4b, 0x65, 0x79}, + []byte{0xeb, 0x9f, 0x77, 0x81, 0xb7, 0x34, 0xca, 0x72, 0xa7, 0x19}, + }, + { + []byte{0x57, 0x69, 0x6b, 0x69}, + []byte{0x60, 0x44, 0xdb, 0x6d, 0x41, 0xb7}, + }, +} + +func TestGolden(t *testing.T) { + for i := 0; i < len(golden); i++ { + g := golden[i] + c, err := NewCipher(g.key) + if err != nil { + t.Errorf("Failed to create cipher at golden index %d", i) + return + } + keystream := make([]byte, len(g.keystream)) + c.XORKeyStream(keystream, keystream) + for j, v := range keystream { + if g.keystream[j] != v { + t.Errorf("Failed at golden index %d", i) + break + } + } + } +} diff --git a/libgo/go/crypto/ripemd160/ripemd160.go b/libgo/go/crypto/ripemd160/ripemd160.go new file mode 100644 index 000000000..5614f1360 --- /dev/null +++ b/libgo/go/crypto/ripemd160/ripemd160.go @@ -0,0 +1,113 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This package implements the RIPEMD-160 hash algorithm. +package ripemd160 + +// RIPEMD-160 is designed by by Hans Dobbertin, Antoon Bosselaers, and Bart +// Preneel with specifications available at: +// http://homes.esat.kuleuven.be/~cosicart/pdf/AB-9601/AB-9601.pdf. + +import ( + "hash" + "os" +) + +// The size of the checksum in bytes. +const Size = 20 + +// The block size of the hash algorithm in bytes. +const BlockSize = 64 + +const ( + _s0 = 0x67452301 + _s1 = 0xefcdab89 + _s2 = 0x98badcfe + _s3 = 0x10325476 + _s4 = 0xc3d2e1f0 +) + +// digest represents the partial evaluation of a checksum. +type digest struct { + s [5]uint32 // running context + x [BlockSize]byte // temporary buffer + nx int // index into x + tc uint64 // total count of bytes processed +} + +func (d *digest) Reset() { + d.s[0], d.s[1], d.s[2], d.s[3], d.s[4] = _s0, _s1, _s2, _s3, _s4 + d.nx = 0 + d.tc = 0 +} + +// New returns a new hash.Hash computing the checksum. +func New() hash.Hash { + result := new(digest) + result.Reset() + return result +} + +func (d *digest) Size() int { return Size } + +func (d *digest) Write(p []byte) (nn int, err os.Error) { + nn = len(p) + d.tc += uint64(nn) + if d.nx > 0 { + n := len(p) + if n > BlockSize-d.nx { + n = BlockSize - d.nx + } + for i := 0; i < n; i++ { + d.x[d.nx+i] = p[i] + } + d.nx += n + if d.nx == BlockSize { + _Block(d, d.x[0:]) + d.nx = 0 + } + p = p[n:] + } + n := _Block(d, p) + p = p[n:] + if len(p) > 0 { + d.nx = copy(d.x[:], p) + } + return +} + +func (d0 *digest) Sum() []byte { + // Make a copy of d0 so that caller can keep writing and summing. + d := new(digest) + *d = *d0 + + // Padding. Add a 1 bit and 0 bits until 56 bytes mod 64. + tc := d.tc + var tmp [64]byte + tmp[0] = 0x80 + if tc%64 < 56 { + d.Write(tmp[0 : 56-tc%64]) + } else { + d.Write(tmp[0 : 64+56-tc%64]) + } + + // Length in bits. + tc <<= 3 + for i := uint(0); i < 8; i++ { + tmp[i] = byte(tc >> (8 * i)) + } + d.Write(tmp[0:8]) + + if d.nx != 0 { + panic("d.nx != 0") + } + + p := make([]byte, 20) + j := 0 + for _, s := range d.s { + p[j], p[j+1], p[j+2], p[j+3] = byte(s), byte(s>>8), byte(s>>16), byte(s>>24) + j += 4 + } + return p +} diff --git a/libgo/go/crypto/ripemd160/ripemd160_test.go b/libgo/go/crypto/ripemd160/ripemd160_test.go new file mode 100644 index 000000000..f4135f5cf --- /dev/null +++ b/libgo/go/crypto/ripemd160/ripemd160_test.go @@ -0,0 +1,64 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ripemd160 + +// Test vectors are from: +// http://homes.esat.kuleuven.be/~bosselae/ripemd160.html + +import ( + "fmt" + "io" + "testing" +) + +type mdTest struct { + out string + in string +} + +var vectors = [...]mdTest{ + {"9c1185a5c5e9fc54612808977ee8f548b2258d31", ""}, + {"0bdc9d2d256b3ee9daae347be6f4dc835a467ffe", "a"}, + {"8eb208f7e05d987a9b044a8e98c6b087f15a0bfc", "abc"}, + {"5d0689ef49d2fae572b881b123a85ffa21595f36", "message digest"}, + {"f71c27109c692c1b56bbdceb5b9d2865b3708dbc", "abcdefghijklmnopqrstuvwxyz"}, + {"12a053384a9c0c88e405a06c27dcf49ada62eb2b", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"}, + {"b0e20b6e3116640286ed3a87a5713079b21f5189", "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"}, + {"9b752e45573d4b39f4dbd3323cab82bf63326bfb", "12345678901234567890123456789012345678901234567890123456789012345678901234567890"}, +} + +func TestVectors(t *testing.T) { + for i := 0; i < len(vectors); i++ { + tv := vectors[i] + md := New() + for j := 0; j < 3; j++ { + if j < 2 { + io.WriteString(md, tv.in) + } else { + io.WriteString(md, tv.in[0:len(tv.in)/2]) + md.Sum() + io.WriteString(md, tv.in[len(tv.in)/2:]) + } + s := fmt.Sprintf("%x", md.Sum()) + if s != tv.out { + t.Fatalf("RIPEMD-160[%d](%s) = %s, expected %s", j, tv.in, s, tv.out) + } + md.Reset() + } + } +} + +func TestMillionA(t *testing.T) { + md := New() + for i := 0; i < 100000; i++ { + io.WriteString(md, "aaaaaaaaaa") + } + out := "52783243c1697bdbe16d37f97f68f08325dc1528" + s := fmt.Sprintf("%x", md.Sum()) + if s != out { + t.Fatalf("RIPEMD-160 (1 million 'a') = %s, expected %s", s, out) + } + md.Reset() +} diff --git a/libgo/go/crypto/ripemd160/ripemd160block.go b/libgo/go/crypto/ripemd160/ripemd160block.go new file mode 100644 index 000000000..7bc8e6c48 --- /dev/null +++ b/libgo/go/crypto/ripemd160/ripemd160block.go @@ -0,0 +1,161 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// RIPEMD-160 block step. +// In its own file so that a faster assembly or C version +// can be substituted easily. + +package ripemd160 + +// work buffer indices and roll amounts for one line +var _n = [80]uint{ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 7, 4, 13, 1, 10, 6, 15, 3, 12, 0, 9, 5, 2, 14, 11, 8, + 3, 10, 14, 4, 9, 15, 8, 1, 2, 7, 0, 6, 13, 11, 5, 12, + 1, 9, 11, 10, 0, 8, 12, 4, 13, 3, 7, 15, 14, 5, 6, 2, + 4, 0, 5, 9, 7, 12, 2, 10, 14, 1, 3, 8, 11, 6, 15, 13, +} + +var _r = [80]uint{ + 11, 14, 15, 12, 5, 8, 7, 9, 11, 13, 14, 15, 6, 7, 9, 8, + 7, 6, 8, 13, 11, 9, 7, 15, 7, 12, 15, 9, 11, 7, 13, 12, + 11, 13, 6, 7, 14, 9, 13, 15, 14, 8, 13, 6, 5, 12, 7, 5, + 11, 12, 14, 15, 14, 15, 9, 8, 9, 14, 5, 6, 8, 6, 5, 12, + 9, 15, 5, 11, 6, 8, 13, 12, 5, 12, 13, 14, 11, 8, 5, 6, +} + +// same for the other parallel one +var n_ = [80]uint{ + 5, 14, 7, 0, 9, 2, 11, 4, 13, 6, 15, 8, 1, 10, 3, 12, + 6, 11, 3, 7, 0, 13, 5, 10, 14, 15, 8, 12, 4, 9, 1, 2, + 15, 5, 1, 3, 7, 14, 6, 9, 11, 8, 12, 2, 10, 0, 4, 13, + 8, 6, 4, 1, 3, 11, 15, 0, 5, 12, 2, 13, 9, 7, 10, 14, + 12, 15, 10, 4, 1, 5, 8, 7, 6, 2, 13, 14, 0, 3, 9, 11, +} + +var r_ = [80]uint{ + 8, 9, 9, 11, 13, 15, 15, 5, 7, 7, 8, 11, 14, 14, 12, 6, + 9, 13, 15, 7, 12, 8, 9, 11, 7, 7, 12, 7, 6, 15, 13, 11, + 9, 7, 15, 11, 8, 6, 6, 14, 12, 13, 5, 14, 13, 13, 7, 5, + 15, 5, 8, 11, 14, 14, 6, 14, 6, 9, 12, 9, 12, 5, 15, 8, + 8, 5, 12, 9, 12, 5, 14, 6, 8, 13, 6, 5, 15, 13, 11, 11, +} + +func _Block(md *digest, p []byte) int { + n := 0 + var x [16]uint32 + var alpha, beta uint32 + for len(p) >= BlockSize { + a, b, c, d, e := md.s[0], md.s[1], md.s[2], md.s[3], md.s[4] + aa, bb, cc, dd, ee := a, b, c, d, e + j := 0 + for i := 0; i < 16; i++ { + x[i] = uint32(p[j]) | uint32(p[j+1])<<8 | uint32(p[j+2])<<16 | uint32(p[j+3])<<24 + j += 4 + } + + // round 1 + i := 0 + for i < 16 { + alpha = a + (b ^ c ^ d) + x[_n[i]] + s := _r[i] + alpha = (alpha<>(32-s)) + e + beta = c<<10 | c>>22 + a, b, c, d, e = e, alpha, b, beta, d + + // parallel line + alpha = aa + (bb ^ (cc | ^dd)) + x[n_[i]] + 0x50a28be6 + s = r_[i] + alpha = (alpha<>(32-s)) + ee + beta = cc<<10 | cc>>22 + aa, bb, cc, dd, ee = ee, alpha, bb, beta, dd + + i++ + } + + // round 2 + for i < 32 { + alpha = a + (b&c | ^b&d) + x[_n[i]] + 0x5a827999 + s := _r[i] + alpha = (alpha<>(32-s)) + e + beta = c<<10 | c>>22 + a, b, c, d, e = e, alpha, b, beta, d + + // parallel line + alpha = aa + (bb&dd | cc&^dd) + x[n_[i]] + 0x5c4dd124 + s = r_[i] + alpha = (alpha<>(32-s)) + ee + beta = cc<<10 | cc>>22 + aa, bb, cc, dd, ee = ee, alpha, bb, beta, dd + + i++ + } + + // round 3 + for i < 48 { + alpha = a + (b | ^c ^ d) + x[_n[i]] + 0x6ed9eba1 + s := _r[i] + alpha = (alpha<>(32-s)) + e + beta = c<<10 | c>>22 + a, b, c, d, e = e, alpha, b, beta, d + + // parallel line + alpha = aa + (bb | ^cc ^ dd) + x[n_[i]] + 0x6d703ef3 + s = r_[i] + alpha = (alpha<>(32-s)) + ee + beta = cc<<10 | cc>>22 + aa, bb, cc, dd, ee = ee, alpha, bb, beta, dd + + i++ + } + + // round 4 + for i < 64 { + alpha = a + (b&d | c&^d) + x[_n[i]] + 0x8f1bbcdc + s := _r[i] + alpha = (alpha<>(32-s)) + e + beta = c<<10 | c>>22 + a, b, c, d, e = e, alpha, b, beta, d + + // parallel line + alpha = aa + (bb&cc | ^bb&dd) + x[n_[i]] + 0x7a6d76e9 + s = r_[i] + alpha = (alpha<>(32-s)) + ee + beta = cc<<10 | cc>>22 + aa, bb, cc, dd, ee = ee, alpha, bb, beta, dd + + i++ + } + + // round 5 + for i < 80 { + alpha = a + (b ^ (c | ^d)) + x[_n[i]] + 0xa953fd4e + s := _r[i] + alpha = (alpha<>(32-s)) + e + beta = c<<10 | c>>22 + a, b, c, d, e = e, alpha, b, beta, d + + // parallel line + alpha = aa + (bb ^ cc ^ dd) + x[n_[i]] + s = r_[i] + alpha = (alpha<>(32-s)) + ee + beta = cc<<10 | cc>>22 + aa, bb, cc, dd, ee = ee, alpha, bb, beta, dd + + i++ + } + + // combine results + dd += c + md.s[1] + md.s[1] = md.s[2] + d + ee + md.s[2] = md.s[3] + e + aa + md.s[3] = md.s[4] + a + bb + md.s[4] = md.s[0] + b + cc + md.s[0] = dd + + p = p[BlockSize:] + n += BlockSize + } + return n +} diff --git a/libgo/go/crypto/rsa/pkcs1v15.go b/libgo/go/crypto/rsa/pkcs1v15.go new file mode 100644 index 000000000..714046250 --- /dev/null +++ b/libgo/go/crypto/rsa/pkcs1v15.go @@ -0,0 +1,273 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package rsa + +import ( + "big" + "crypto/subtle" + "io" + "os" +) + +// This file implements encryption and decryption using PKCS#1 v1.5 padding. + +// EncryptPKCS1v15 encrypts the given message with RSA and the padding scheme from PKCS#1 v1.5. +// The message must be no longer than the length of the public modulus minus 11 bytes. +// WARNING: use of this function to encrypt plaintexts other than session keys +// is dangerous. Use RSA OAEP in new protocols. +func EncryptPKCS1v15(rand io.Reader, pub *PublicKey, msg []byte) (out []byte, err os.Error) { + k := (pub.N.BitLen() + 7) / 8 + if len(msg) > k-11 { + err = MessageTooLongError{} + return + } + + // EM = 0x02 || PS || 0x00 || M + em := make([]byte, k-1) + em[0] = 2 + ps, mm := em[1:len(em)-len(msg)-1], em[len(em)-len(msg):] + err = nonZeroRandomBytes(ps, rand) + if err != nil { + return + } + em[len(em)-len(msg)-1] = 0 + copy(mm, msg) + + m := new(big.Int).SetBytes(em) + c := encrypt(new(big.Int), pub, m) + out = c.Bytes() + return +} + +// DecryptPKCS1v15 decrypts a plaintext using RSA and the padding scheme from PKCS#1 v1.5. +// If rand != nil, it uses RSA blinding to avoid timing side-channel attacks. +func DecryptPKCS1v15(rand io.Reader, priv *PrivateKey, ciphertext []byte) (out []byte, err os.Error) { + valid, out, err := decryptPKCS1v15(rand, priv, ciphertext) + if err == nil && valid == 0 { + err = DecryptionError{} + } + + return +} + +// DecryptPKCS1v15SessionKey decrypts a session key using RSA and the padding scheme from PKCS#1 v1.5. +// If rand != nil, it uses RSA blinding to avoid timing side-channel attacks. +// It returns an error if the ciphertext is the wrong length or if the +// ciphertext is greater than the public modulus. Otherwise, no error is +// returned. If the padding is valid, the resulting plaintext message is copied +// into key. Otherwise, key is unchanged. These alternatives occur in constant +// time. It is intended that the user of this function generate a random +// session key beforehand and continue the protocol with the resulting value. +// This will remove any possibility that an attacker can learn any information +// about the plaintext. +// See ``Chosen Ciphertext Attacks Against Protocols Based on the RSA +// Encryption Standard PKCS #1'', Daniel Bleichenbacher, Advances in Cryptology +// (Crypto '98), +func DecryptPKCS1v15SessionKey(rand io.Reader, priv *PrivateKey, ciphertext []byte, key []byte) (err os.Error) { + k := (priv.N.BitLen() + 7) / 8 + if k-(len(key)+3+8) < 0 { + err = DecryptionError{} + return + } + + valid, msg, err := decryptPKCS1v15(rand, priv, ciphertext) + if err != nil { + return + } + + valid &= subtle.ConstantTimeEq(int32(len(msg)), int32(len(key))) + subtle.ConstantTimeCopy(valid, key, msg) + return +} + +func decryptPKCS1v15(rand io.Reader, priv *PrivateKey, ciphertext []byte) (valid int, msg []byte, err os.Error) { + k := (priv.N.BitLen() + 7) / 8 + if k < 11 { + err = DecryptionError{} + return + } + + c := new(big.Int).SetBytes(ciphertext) + m, err := decrypt(rand, priv, c) + if err != nil { + return + } + + em := leftPad(m.Bytes(), k) + firstByteIsZero := subtle.ConstantTimeByteEq(em[0], 0) + secondByteIsTwo := subtle.ConstantTimeByteEq(em[1], 2) + + // The remainder of the plaintext must be a string of non-zero random + // octets, followed by a 0, followed by the message. + // lookingForIndex: 1 iff we are still looking for the zero. + // index: the offset of the first zero byte. + var lookingForIndex, index int + lookingForIndex = 1 + + for i := 2; i < len(em); i++ { + equals0 := subtle.ConstantTimeByteEq(em[i], 0) + index = subtle.ConstantTimeSelect(lookingForIndex&equals0, i, index) + lookingForIndex = subtle.ConstantTimeSelect(equals0, 0, lookingForIndex) + } + + valid = firstByteIsZero & secondByteIsTwo & (^lookingForIndex & 1) + msg = em[index+1:] + return +} + +// nonZeroRandomBytes fills the given slice with non-zero random octets. +func nonZeroRandomBytes(s []byte, rand io.Reader) (err os.Error) { + _, err = io.ReadFull(rand, s) + if err != nil { + return + } + + for i := 0; i < len(s); i++ { + for s[i] == 0 { + _, err = rand.Read(s[i : i+1]) + if err != nil { + return + } + // In tests, the PRNG may return all zeros so we do + // this to break the loop. + s[i] ^= 0x42 + } + } + + return +} + +// Due to the design of PKCS#1 v1.5, we need to know the exact hash function in +// use. A generic hash.Hash will not do. +type PKCS1v15Hash int + +const ( + HashMD5 PKCS1v15Hash = iota + HashSHA1 + HashSHA256 + HashSHA384 + HashSHA512 + HashMD5SHA1 // combined MD5 and SHA1 hash used for RSA signing in TLS. +) + +// These are ASN1 DER structures: +// DigestInfo ::= SEQUENCE { +// digestAlgorithm AlgorithmIdentifier, +// digest OCTET STRING +// } +// For performance, we don't use the generic ASN1 encoder. Rather, we +// precompute a prefix of the digest value that makes a valid ASN1 DER string +// with the correct contents. +var hashPrefixes = [][]byte{ + // HashMD5 + {0x30, 0x20, 0x30, 0x0c, 0x06, 0x08, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x02, 0x05, 0x05, 0x00, 0x04, 0x10}, + // HashSHA1 + {0x30, 0x21, 0x30, 0x09, 0x06, 0x05, 0x2b, 0x0e, 0x03, 0x02, 0x1a, 0x05, 0x00, 0x04, 0x14}, + // HashSHA256 + {0x30, 0x31, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x01, 0x05, 0x00, 0x04, 0x20}, + // HashSHA384 + {0x30, 0x41, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x02, 0x05, 0x00, 0x04, 0x30}, + // HashSHA512 + {0x30, 0x51, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x03, 0x05, 0x00, 0x04, 0x40}, + // HashMD5SHA1 + {}, // A special TLS case which doesn't use an ASN1 prefix. +} + +// SignPKCS1v15 calcuates the signature of hashed using RSASSA-PSS-SIGN from RSA PKCS#1 v1.5. +// Note that hashed must be the result of hashing the input message using the +// given hash function. +func SignPKCS1v15(rand io.Reader, priv *PrivateKey, hash PKCS1v15Hash, hashed []byte) (s []byte, err os.Error) { + hashLen, prefix, err := pkcs1v15HashInfo(hash, len(hashed)) + if err != nil { + return + } + + tLen := len(prefix) + hashLen + k := (priv.N.BitLen() + 7) / 8 + if k < tLen+11 { + return nil, MessageTooLongError{} + } + + // EM = 0x00 || 0x01 || PS || 0x00 || T + em := make([]byte, k) + em[1] = 1 + for i := 2; i < k-tLen-1; i++ { + em[i] = 0xff + } + copy(em[k-tLen:k-hashLen], prefix) + copy(em[k-hashLen:k], hashed) + + m := new(big.Int).SetBytes(em) + c, err := decrypt(rand, priv, m) + if err == nil { + s = c.Bytes() + } + return +} + +// VerifyPKCS1v15 verifies an RSA PKCS#1 v1.5 signature. +// hashed is the result of hashing the input message using the given hash +// function and sig is the signature. A valid signature is indicated by +// returning a nil error. +func VerifyPKCS1v15(pub *PublicKey, hash PKCS1v15Hash, hashed []byte, sig []byte) (err os.Error) { + hashLen, prefix, err := pkcs1v15HashInfo(hash, len(hashed)) + if err != nil { + return + } + + tLen := len(prefix) + hashLen + k := (pub.N.BitLen() + 7) / 8 + if k < tLen+11 { + err = VerificationError{} + return + } + + c := new(big.Int).SetBytes(sig) + m := encrypt(new(big.Int), pub, c) + em := leftPad(m.Bytes(), k) + // EM = 0x00 || 0x01 || PS || 0x00 || T + + ok := subtle.ConstantTimeByteEq(em[0], 0) + ok &= subtle.ConstantTimeByteEq(em[1], 1) + ok &= subtle.ConstantTimeCompare(em[k-hashLen:k], hashed) + ok &= subtle.ConstantTimeCompare(em[k-tLen:k-hashLen], prefix) + ok &= subtle.ConstantTimeByteEq(em[k-tLen-1], 0) + + for i := 2; i < k-tLen-1; i++ { + ok &= subtle.ConstantTimeByteEq(em[i], 0xff) + } + + if ok != 1 { + return VerificationError{} + } + + return nil +} + +func pkcs1v15HashInfo(hash PKCS1v15Hash, inLen int) (hashLen int, prefix []byte, err os.Error) { + switch hash { + case HashMD5: + hashLen = 16 + case HashSHA1: + hashLen = 20 + case HashSHA256: + hashLen = 32 + case HashSHA384: + hashLen = 48 + case HashSHA512: + hashLen = 64 + case HashMD5SHA1: + hashLen = 36 + default: + return 0, nil, os.ErrorString("unknown hash function") + } + + if inLen != hashLen { + return 0, nil, os.ErrorString("input must be hashed message") + } + + prefix = hashPrefixes[int(hash)] + return +} diff --git a/libgo/go/crypto/rsa/pkcs1v15_test.go b/libgo/go/crypto/rsa/pkcs1v15_test.go new file mode 100644 index 000000000..bf6306dc2 --- /dev/null +++ b/libgo/go/crypto/rsa/pkcs1v15_test.go @@ -0,0 +1,220 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package rsa + +import ( + "big" + "bytes" + "crypto/rand" + "crypto/sha1" + "encoding/base64" + "encoding/hex" + "io" + "testing" + "testing/quick" +) + +func decodeBase64(in string) []byte { + out := make([]byte, base64.StdEncoding.DecodedLen(len(in))) + n, err := base64.StdEncoding.Decode(out, []byte(in)) + if err != nil { + return nil + } + return out[0:n] +} + +type DecryptPKCS1v15Test struct { + in, out string +} + +// These test vectors were generated with `openssl rsautl -pkcs -encrypt` +var decryptPKCS1v15Tests = []DecryptPKCS1v15Test{ + { + "gIcUIoVkD6ATMBk/u/nlCZCCWRKdkfjCgFdo35VpRXLduiKXhNz1XupLLzTXAybEq15juc+EgY5o0DHv/nt3yg==", + "x", + }, + { + "Y7TOCSqofGhkRb+jaVRLzK8xw2cSo1IVES19utzv6hwvx+M8kFsoWQm5DzBeJCZTCVDPkTpavUuEbgp8hnUGDw==", + "testing.", + }, + { + "arReP9DJtEVyV2Dg3dDp4c/PSk1O6lxkoJ8HcFupoRorBZG+7+1fDAwT1olNddFnQMjmkb8vxwmNMoTAT/BFjQ==", + "testing.\n", + }, + { + "WtaBXIoGC54+vH0NH0CHHE+dRDOsMc/6BrfFu2lEqcKL9+uDuWaf+Xj9mrbQCjjZcpQuX733zyok/jsnqe/Ftw==", + "01234567890123456789012345678901234567890123456789012", + }, +} + +func TestDecryptPKCS1v15(t *testing.T) { + for i, test := range decryptPKCS1v15Tests { + out, err := DecryptPKCS1v15(nil, rsaPrivateKey, decodeBase64(test.in)) + if err != nil { + t.Errorf("#%d error decrypting", i) + } + want := []byte(test.out) + if bytes.Compare(out, want) != 0 { + t.Errorf("#%d got:%#v want:%#v", i, out, want) + } + } +} + +func TestEncryptPKCS1v15(t *testing.T) { + random := rand.Reader + k := (rsaPrivateKey.N.BitLen() + 7) / 8 + + tryEncryptDecrypt := func(in []byte, blind bool) bool { + if len(in) > k-11 { + in = in[0 : k-11] + } + + ciphertext, err := EncryptPKCS1v15(random, &rsaPrivateKey.PublicKey, in) + if err != nil { + t.Errorf("error encrypting: %s", err) + return false + } + + var rand io.Reader + if !blind { + rand = nil + } else { + rand = random + } + plaintext, err := DecryptPKCS1v15(rand, rsaPrivateKey, ciphertext) + if err != nil { + t.Errorf("error decrypting: %s", err) + return false + } + + if bytes.Compare(plaintext, in) != 0 { + t.Errorf("output mismatch: %#v %#v", plaintext, in) + return false + } + return true + } + + quick.Check(tryEncryptDecrypt, nil) +} + +// These test vectors were generated with `openssl rsautl -pkcs -encrypt` +var decryptPKCS1v15SessionKeyTests = []DecryptPKCS1v15Test{ + { + "e6ukkae6Gykq0fKzYwULpZehX+UPXYzMoB5mHQUDEiclRbOTqas4Y0E6nwns1BBpdvEJcilhl5zsox/6DtGsYg==", + "1234", + }, + { + "Dtis4uk/q/LQGGqGk97P59K03hkCIVFMEFZRgVWOAAhxgYpCRG0MX2adptt92l67IqMki6iVQyyt0TtX3IdtEw==", + "FAIL", + }, + { + "LIyFyCYCptPxrvTxpol8F3M7ZivlMsf53zs0vHRAv+rDIh2YsHS69ePMoPMe3TkOMZ3NupiL3takPxIs1sK+dw==", + "abcd", + }, + { + "bafnobel46bKy76JzqU/RIVOH0uAYvzUtauKmIidKgM0sMlvobYVAVQPeUQ/oTGjbIZ1v/6Gyi5AO4DtHruGdw==", + "FAIL", + }, +} + +func TestEncryptPKCS1v15SessionKey(t *testing.T) { + for i, test := range decryptPKCS1v15SessionKeyTests { + key := []byte("FAIL") + err := DecryptPKCS1v15SessionKey(nil, rsaPrivateKey, decodeBase64(test.in), key) + if err != nil { + t.Errorf("#%d error decrypting", i) + } + want := []byte(test.out) + if bytes.Compare(key, want) != 0 { + t.Errorf("#%d got:%#v want:%#v", i, key, want) + } + } +} + +func TestNonZeroRandomBytes(t *testing.T) { + random := rand.Reader + + b := make([]byte, 512) + err := nonZeroRandomBytes(b, random) + if err != nil { + t.Errorf("returned error: %s", err) + } + for _, b := range b { + if b == 0 { + t.Errorf("Zero octet found") + return + } + } +} + +type signPKCS1v15Test struct { + in, out string +} + +// These vectors have been tested with +// `openssl rsautl -verify -inkey pk -in signature | hexdump -C` +var signPKCS1v15Tests = []signPKCS1v15Test{ + {"Test.\n", "a4f3fa6ea93bcdd0c57be020c1193ecbfd6f200a3d95c409769b029578fa0e336ad9a347600e40d3ae823b8c7e6bad88cc07c1d54c3a1523cbbb6d58efc362ae"}, +} + +func TestSignPKCS1v15(t *testing.T) { + for i, test := range signPKCS1v15Tests { + h := sha1.New() + h.Write([]byte(test.in)) + digest := h.Sum() + + s, err := SignPKCS1v15(nil, rsaPrivateKey, HashSHA1, digest) + if err != nil { + t.Errorf("#%d %s", i, err) + } + + expected, _ := hex.DecodeString(test.out) + if bytes.Compare(s, expected) != 0 { + t.Errorf("#%d got: %x want: %x", i, s, expected) + } + } +} + +func TestVerifyPKCS1v15(t *testing.T) { + for i, test := range signPKCS1v15Tests { + h := sha1.New() + h.Write([]byte(test.in)) + digest := h.Sum() + + sig, _ := hex.DecodeString(test.out) + + err := VerifyPKCS1v15(&rsaPrivateKey.PublicKey, HashSHA1, digest, sig) + if err != nil { + t.Errorf("#%d %s", i, err) + } + } +} + +func bigFromString(s string) *big.Int { + ret := new(big.Int) + ret.SetString(s, 10) + return ret +} + +// In order to generate new test vectors you'll need the PEM form of this key: +// -----BEGIN RSA PRIVATE KEY----- +// MIIBOgIBAAJBALKZD0nEffqM1ACuak0bijtqE2QrI/KLADv7l3kK3ppMyCuLKoF0 +// fd7Ai2KW5ToIwzFofvJcS/STa6HA5gQenRUCAwEAAQJBAIq9amn00aS0h/CrjXqu +// /ThglAXJmZhOMPVn4eiu7/ROixi9sex436MaVeMqSNf7Ex9a8fRNfWss7Sqd9eWu +// RTUCIQDasvGASLqmjeffBNLTXV2A5g4t+kLVCpsEIZAycV5GswIhANEPLmax0ME/ +// EO+ZJ79TJKN5yiGBRsv5yvx5UiHxajEXAiAhAol5N4EUyq6I9w1rYdhPMGpLfk7A +// IU2snfRJ6Nq2CQIgFrPsWRCkV+gOYcajD17rEqmuLrdIRexpg8N1DOSXoJ8CIGlS +// tAboUGBxTDq3ZroNism3DaMIbKPyYrAqhKov1h5V +// -----END RSA PRIVATE KEY----- + +var rsaPrivateKey = &PrivateKey{ + PublicKey: PublicKey{ + N: bigFromString("9353930466774385905609975137998169297361893554149986716853295022578535724979677252958524466350471210367835187480748268864277464700638583474144061408845077"), + E: 65537, + }, + D: bigFromString("7266398431328116344057699379749222532279343923819063639497049039389899328538543087657733766554155839834519529439851673014800261285757759040931985506583861"), + P: bigFromString("98920366548084643601728869055592650835572950932266967461790948584315647051443"), + Q: bigFromString("94560208308847015747498523884063394671606671904944666360068158221458669711639"), +} diff --git a/libgo/go/crypto/rsa/rsa.go b/libgo/go/crypto/rsa/rsa.go new file mode 100644 index 000000000..c7a8d2053 --- /dev/null +++ b/libgo/go/crypto/rsa/rsa.go @@ -0,0 +1,445 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This package implements RSA encryption as specified in PKCS#1. +package rsa + +// TODO(agl): Add support for PSS padding. + +import ( + "big" + "crypto/subtle" + "hash" + "io" + "os" +) + +var bigZero = big.NewInt(0) +var bigOne = big.NewInt(1) + +// randomPrime returns a number, p, of the given size, such that p is prime +// with high probability. +func randomPrime(rand io.Reader, bits int) (p *big.Int, err os.Error) { + if bits < 1 { + err = os.EINVAL + } + + bytes := make([]byte, (bits+7)/8) + p = new(big.Int) + + for { + _, err = io.ReadFull(rand, bytes) + if err != nil { + return + } + + // Don't let the value be too small. + bytes[0] |= 0x80 + // Make the value odd since an even number this large certainly isn't prime. + bytes[len(bytes)-1] |= 1 + + p.SetBytes(bytes) + if big.ProbablyPrime(p, 20) { + return + } + } + + return +} + +// randomNumber returns a uniform random value in [0, max). +func randomNumber(rand io.Reader, max *big.Int) (n *big.Int, err os.Error) { + k := (max.BitLen() + 7) / 8 + + // r is the number of bits in the used in the most significant byte of + // max. + r := uint(max.BitLen() % 8) + if r == 0 { + r = 8 + } + + bytes := make([]byte, k) + n = new(big.Int) + + for { + _, err = io.ReadFull(rand, bytes) + if err != nil { + return + } + + // Clear bits in the first byte to increase the probability + // that the candidate is < max. + bytes[0] &= uint8(int(1< k-2*hash.Size()-2 { + err = MessageTooLongError{} + return + } + + hash.Write(label) + lHash := hash.Sum() + hash.Reset() + + em := make([]byte, k) + seed := em[1 : 1+hash.Size()] + db := em[1+hash.Size():] + + copy(db[0:hash.Size()], lHash) + db[len(db)-len(msg)-1] = 1 + copy(db[len(db)-len(msg):], msg) + + _, err = io.ReadFull(rand, seed) + if err != nil { + return + } + + mgf1XOR(db, hash, seed) + mgf1XOR(seed, hash, db) + + m := new(big.Int) + m.SetBytes(em) + c := encrypt(new(big.Int), pub, m) + out = c.Bytes() + return +} + +// A DecryptionError represents a failure to decrypt a message. +// It is deliberately vague to avoid adaptive attacks. +type DecryptionError struct{} + +func (DecryptionError) String() string { return "RSA decryption error" } + +// A VerificationError represents a failure to verify a signature. +// It is deliberately vague to avoid adaptive attacks. +type VerificationError struct{} + +func (VerificationError) String() string { return "RSA verification error" } + +// modInverse returns ia, the inverse of a in the multiplicative group of prime +// order n. It requires that a be a member of the group (i.e. less than n). +func modInverse(a, n *big.Int) (ia *big.Int, ok bool) { + g := new(big.Int) + x := new(big.Int) + y := new(big.Int) + big.GcdInt(g, x, y, a, n) + if g.Cmp(bigOne) != 0 { + // In this case, a and n aren't coprime and we cannot calculate + // the inverse. This happens because the values of n are nearly + // prime (being the product of two primes) rather than truly + // prime. + return + } + + if x.Cmp(bigOne) < 0 { + // 0 is not the multiplicative inverse of any element so, if x + // < 1, then x is negative. + x.Add(x, n) + } + + return x, true +} + +// decrypt performs an RSA decryption, resulting in a plaintext integer. If a +// random source is given, RSA blinding is used. +func decrypt(rand io.Reader, priv *PrivateKey, c *big.Int) (m *big.Int, err os.Error) { + // TODO(agl): can we get away with reusing blinds? + if c.Cmp(priv.N) > 0 { + err = DecryptionError{} + return + } + + var ir *big.Int + if rand != nil { + // Blinding enabled. Blinding involves multiplying c by r^e. + // Then the decryption operation performs (m^e * r^e)^d mod n + // which equals mr mod n. The factor of r can then be removed + // by multipling by the multiplicative inverse of r. + + var r *big.Int + + for { + r, err = randomNumber(rand, priv.N) + if err != nil { + return + } + if r.Cmp(bigZero) == 0 { + r = bigOne + } + var ok bool + ir, ok = modInverse(r, priv.N) + if ok { + break + } + } + bigE := big.NewInt(int64(priv.E)) + rpowe := new(big.Int).Exp(r, bigE, priv.N) + c.Mul(c, rpowe) + c.Mod(c, priv.N) + } + + m = new(big.Int).Exp(c, priv.D, priv.N) + + if ir != nil { + // Unblind. + m.Mul(m, ir) + m.Mod(m, priv.N) + } + + return +} + +// DecryptOAEP decrypts ciphertext using RSA-OAEP. +// If rand != nil, DecryptOAEP uses RSA blinding to avoid timing side-channel attacks. +func DecryptOAEP(hash hash.Hash, rand io.Reader, priv *PrivateKey, ciphertext []byte, label []byte) (msg []byte, err os.Error) { + k := (priv.N.BitLen() + 7) / 8 + if len(ciphertext) > k || + k < hash.Size()*2+2 { + err = DecryptionError{} + return + } + + c := new(big.Int).SetBytes(ciphertext) + + m, err := decrypt(rand, priv, c) + if err != nil { + return + } + + hash.Write(label) + lHash := hash.Sum() + hash.Reset() + + // Converting the plaintext number to bytes will strip any + // leading zeros so we may have to left pad. We do this unconditionally + // to avoid leaking timing information. (Although we still probably + // leak the number of leading zeros. It's not clear that we can do + // anything about this.) + em := leftPad(m.Bytes(), k) + + firstByteIsZero := subtle.ConstantTimeByteEq(em[0], 0) + + seed := em[1 : hash.Size()+1] + db := em[hash.Size()+1:] + + mgf1XOR(seed, hash, db) + mgf1XOR(db, hash, seed) + + lHash2 := db[0:hash.Size()] + + // We have to validate the plaintext in constant time in order to avoid + // attacks like: J. Manger. A Chosen Ciphertext Attack on RSA Optimal + // Asymmetric Encryption Padding (OAEP) as Standardized in PKCS #1 + // v2.0. In J. Kilian, editor, Advances in Cryptology. + lHash2Good := subtle.ConstantTimeCompare(lHash, lHash2) + + // The remainder of the plaintext must be zero or more 0x00, followed + // by 0x01, followed by the message. + // lookingForIndex: 1 iff we are still looking for the 0x01 + // index: the offset of the first 0x01 byte + // invalid: 1 iff we saw a non-zero byte before the 0x01. + var lookingForIndex, index, invalid int + lookingForIndex = 1 + rest := db[hash.Size():] + + for i := 0; i < len(rest); i++ { + equals0 := subtle.ConstantTimeByteEq(rest[i], 0) + equals1 := subtle.ConstantTimeByteEq(rest[i], 1) + index = subtle.ConstantTimeSelect(lookingForIndex&equals1, i, index) + lookingForIndex = subtle.ConstantTimeSelect(equals1, 0, lookingForIndex) + invalid = subtle.ConstantTimeSelect(lookingForIndex&^equals0, 1, invalid) + } + + if firstByteIsZero&lHash2Good&^invalid&^lookingForIndex != 1 { + err = DecryptionError{} + return + } + + msg = rest[index+1:] + return +} + +// leftPad returns a new slice of length size. The contents of input are right +// aligned in the new slice. +func leftPad(input []byte, size int) (out []byte) { + n := len(input) + if n > size { + n = size + } + out = make([]byte, size) + copy(out[len(out)-n:], input) + return +} diff --git a/libgo/go/crypto/rsa/rsa_test.go b/libgo/go/crypto/rsa/rsa_test.go new file mode 100644 index 000000000..df1f17f17 --- /dev/null +++ b/libgo/go/crypto/rsa/rsa_test.go @@ -0,0 +1,250 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package rsa + +import ( + "big" + "bytes" + "crypto/rand" + "crypto/sha1" + "testing" +) + +func TestKeyGeneration(t *testing.T) { + random := rand.Reader + + priv, err := GenerateKey(random, 1024) + if err != nil { + t.Errorf("failed to generate key") + } + pub := &priv.PublicKey + m := big.NewInt(42) + c := encrypt(new(big.Int), pub, m) + m2, err := decrypt(nil, priv, c) + if err != nil { + t.Errorf("error while decrypting: %s", err) + } + if m.Cmp(m2) != 0 { + t.Errorf("got:%v, want:%v (%s)", m2, m, priv) + } + + m3, err := decrypt(random, priv, c) + if err != nil { + t.Errorf("error while decrypting (blind): %s", err) + } + if m.Cmp(m3) != 0 { + t.Errorf("(blind) got:%v, want:%v", m3, m) + } +} + +type testEncryptOAEPMessage struct { + in []byte + seed []byte + out []byte +} + +type testEncryptOAEPStruct struct { + modulus string + e int + d string + msgs []testEncryptOAEPMessage +} + +func TestEncryptOAEP(t *testing.T) { + sha1 := sha1.New() + n := new(big.Int) + for i, test := range testEncryptOAEPData { + n.SetString(test.modulus, 16) + public := PublicKey{n, test.e} + + for j, message := range test.msgs { + randomSource := bytes.NewBuffer(message.seed) + out, err := EncryptOAEP(sha1, randomSource, &public, message.in, nil) + if err != nil { + t.Errorf("#%d,%d error: %s", i, j, err) + } + if bytes.Compare(out, message.out) != 0 { + t.Errorf("#%d,%d bad result: %s (want %s)", i, j, out, message.out) + } + } + } +} + +func TestDecryptOAEP(t *testing.T) { + random := rand.Reader + + sha1 := sha1.New() + n := new(big.Int) + d := new(big.Int) + for i, test := range testEncryptOAEPData { + n.SetString(test.modulus, 16) + d.SetString(test.d, 16) + private := PrivateKey{PublicKey{n, test.e}, d, nil, nil} + + for j, message := range test.msgs { + out, err := DecryptOAEP(sha1, nil, &private, message.out, nil) + if err != nil { + t.Errorf("#%d,%d error: %s", i, j, err) + } else if bytes.Compare(out, message.in) != 0 { + t.Errorf("#%d,%d bad result: %#v (want %#v)", i, j, out, message.in) + } + + // Decrypt with blinding. + out, err = DecryptOAEP(sha1, random, &private, message.out, nil) + if err != nil { + t.Errorf("#%d,%d (blind) error: %s", i, j, err) + } else if bytes.Compare(out, message.in) != 0 { + t.Errorf("#%d,%d (blind) bad result: %#v (want %#v)", i, j, out, message.in) + } + } + } +} + +// testEncryptOAEPData contains a subset of the vectors from RSA's "Test vectors for RSA-OAEP". +var testEncryptOAEPData = []testEncryptOAEPStruct{ + // Key 1 + {"a8b3b284af8eb50b387034a860f146c4919f318763cd6c5598c8ae4811a1e0abc4c7e0b082d693a5e7fced675cf4668512772c0cbc64a742c6c630f533c8cc72f62ae833c40bf25842e984bb78bdbf97c0107d55bdb662f5c4e0fab9845cb5148ef7392dd3aaff93ae1e6b667bb3d4247616d4f5ba10d4cfd226de88d39f16fb", + 65537, + "53339cfdb79fc8466a655c7316aca85c55fd8f6dd898fdaf119517ef4f52e8fd8e258df93fee180fa0e4ab29693cd83b152a553d4ac4d1812b8b9fa5af0e7f55fe7304df41570926f3311f15c4d65a732c483116ee3d3d2d0af3549ad9bf7cbfb78ad884f84d5beb04724dc7369b31def37d0cf539e9cfcdd3de653729ead5d1", + []testEncryptOAEPMessage{ + // Example 1.1 + { + []byte{0x66, 0x28, 0x19, 0x4e, 0x12, 0x07, 0x3d, 0xb0, + 0x3b, 0xa9, 0x4c, 0xda, 0x9e, 0xf9, 0x53, 0x23, 0x97, + 0xd5, 0x0d, 0xba, 0x79, 0xb9, 0x87, 0x00, 0x4a, 0xfe, + 0xfe, 0x34, + }, + []byte{0x18, 0xb7, 0x76, 0xea, 0x21, 0x06, 0x9d, 0x69, + 0x77, 0x6a, 0x33, 0xe9, 0x6b, 0xad, 0x48, 0xe1, 0xdd, + 0xa0, 0xa5, 0xef, + }, + []byte{0x35, 0x4f, 0xe6, 0x7b, 0x4a, 0x12, 0x6d, 0x5d, + 0x35, 0xfe, 0x36, 0xc7, 0x77, 0x79, 0x1a, 0x3f, 0x7b, + 0xa1, 0x3d, 0xef, 0x48, 0x4e, 0x2d, 0x39, 0x08, 0xaf, + 0xf7, 0x22, 0xfa, 0xd4, 0x68, 0xfb, 0x21, 0x69, 0x6d, + 0xe9, 0x5d, 0x0b, 0xe9, 0x11, 0xc2, 0xd3, 0x17, 0x4f, + 0x8a, 0xfc, 0xc2, 0x01, 0x03, 0x5f, 0x7b, 0x6d, 0x8e, + 0x69, 0x40, 0x2d, 0xe5, 0x45, 0x16, 0x18, 0xc2, 0x1a, + 0x53, 0x5f, 0xa9, 0xd7, 0xbf, 0xc5, 0xb8, 0xdd, 0x9f, + 0xc2, 0x43, 0xf8, 0xcf, 0x92, 0x7d, 0xb3, 0x13, 0x22, + 0xd6, 0xe8, 0x81, 0xea, 0xa9, 0x1a, 0x99, 0x61, 0x70, + 0xe6, 0x57, 0xa0, 0x5a, 0x26, 0x64, 0x26, 0xd9, 0x8c, + 0x88, 0x00, 0x3f, 0x84, 0x77, 0xc1, 0x22, 0x70, 0x94, + 0xa0, 0xd9, 0xfa, 0x1e, 0x8c, 0x40, 0x24, 0x30, 0x9c, + 0xe1, 0xec, 0xcc, 0xb5, 0x21, 0x00, 0x35, 0xd4, 0x7a, + 0xc7, 0x2e, 0x8a, + }, + }, + // Example 1.2 + { + []byte{0x75, 0x0c, 0x40, 0x47, 0xf5, 0x47, 0xe8, 0xe4, + 0x14, 0x11, 0x85, 0x65, 0x23, 0x29, 0x8a, 0xc9, 0xba, + 0xe2, 0x45, 0xef, 0xaf, 0x13, 0x97, 0xfb, 0xe5, 0x6f, + 0x9d, 0xd5, + }, + []byte{0x0c, 0xc7, 0x42, 0xce, 0x4a, 0x9b, 0x7f, 0x32, + 0xf9, 0x51, 0xbc, 0xb2, 0x51, 0xef, 0xd9, 0x25, 0xfe, + 0x4f, 0xe3, 0x5f, + }, + []byte{0x64, 0x0d, 0xb1, 0xac, 0xc5, 0x8e, 0x05, 0x68, + 0xfe, 0x54, 0x07, 0xe5, 0xf9, 0xb7, 0x01, 0xdf, 0xf8, + 0xc3, 0xc9, 0x1e, 0x71, 0x6c, 0x53, 0x6f, 0xc7, 0xfc, + 0xec, 0x6c, 0xb5, 0xb7, 0x1c, 0x11, 0x65, 0x98, 0x8d, + 0x4a, 0x27, 0x9e, 0x15, 0x77, 0xd7, 0x30, 0xfc, 0x7a, + 0x29, 0x93, 0x2e, 0x3f, 0x00, 0xc8, 0x15, 0x15, 0x23, + 0x6d, 0x8d, 0x8e, 0x31, 0x01, 0x7a, 0x7a, 0x09, 0xdf, + 0x43, 0x52, 0xd9, 0x04, 0xcd, 0xeb, 0x79, 0xaa, 0x58, + 0x3a, 0xdc, 0xc3, 0x1e, 0xa6, 0x98, 0xa4, 0xc0, 0x52, + 0x83, 0xda, 0xba, 0x90, 0x89, 0xbe, 0x54, 0x91, 0xf6, + 0x7c, 0x1a, 0x4e, 0xe4, 0x8d, 0xc7, 0x4b, 0xbb, 0xe6, + 0x64, 0x3a, 0xef, 0x84, 0x66, 0x79, 0xb4, 0xcb, 0x39, + 0x5a, 0x35, 0x2d, 0x5e, 0xd1, 0x15, 0x91, 0x2d, 0xf6, + 0x96, 0xff, 0xe0, 0x70, 0x29, 0x32, 0x94, 0x6d, 0x71, + 0x49, 0x2b, 0x44, + }, + }, + // Example 1.3 + { + []byte{0xd9, 0x4a, 0xe0, 0x83, 0x2e, 0x64, 0x45, 0xce, + 0x42, 0x33, 0x1c, 0xb0, 0x6d, 0x53, 0x1a, 0x82, 0xb1, + 0xdb, 0x4b, 0xaa, 0xd3, 0x0f, 0x74, 0x6d, 0xc9, 0x16, + 0xdf, 0x24, 0xd4, 0xe3, 0xc2, 0x45, 0x1f, 0xff, 0x59, + 0xa6, 0x42, 0x3e, 0xb0, 0xe1, 0xd0, 0x2d, 0x4f, 0xe6, + 0x46, 0xcf, 0x69, 0x9d, 0xfd, 0x81, 0x8c, 0x6e, 0x97, + 0xb0, 0x51, + }, + []byte{0x25, 0x14, 0xdf, 0x46, 0x95, 0x75, 0x5a, 0x67, + 0xb2, 0x88, 0xea, 0xf4, 0x90, 0x5c, 0x36, 0xee, 0xc6, + 0x6f, 0xd2, 0xfd, + }, + []byte{0x42, 0x37, 0x36, 0xed, 0x03, 0x5f, 0x60, 0x26, + 0xaf, 0x27, 0x6c, 0x35, 0xc0, 0xb3, 0x74, 0x1b, 0x36, + 0x5e, 0x5f, 0x76, 0xca, 0x09, 0x1b, 0x4e, 0x8c, 0x29, + 0xe2, 0xf0, 0xbe, 0xfe, 0xe6, 0x03, 0x59, 0x5a, 0xa8, + 0x32, 0x2d, 0x60, 0x2d, 0x2e, 0x62, 0x5e, 0x95, 0xeb, + 0x81, 0xb2, 0xf1, 0xc9, 0x72, 0x4e, 0x82, 0x2e, 0xca, + 0x76, 0xdb, 0x86, 0x18, 0xcf, 0x09, 0xc5, 0x34, 0x35, + 0x03, 0xa4, 0x36, 0x08, 0x35, 0xb5, 0x90, 0x3b, 0xc6, + 0x37, 0xe3, 0x87, 0x9f, 0xb0, 0x5e, 0x0e, 0xf3, 0x26, + 0x85, 0xd5, 0xae, 0xc5, 0x06, 0x7c, 0xd7, 0xcc, 0x96, + 0xfe, 0x4b, 0x26, 0x70, 0xb6, 0xea, 0xc3, 0x06, 0x6b, + 0x1f, 0xcf, 0x56, 0x86, 0xb6, 0x85, 0x89, 0xaa, 0xfb, + 0x7d, 0x62, 0x9b, 0x02, 0xd8, 0xf8, 0x62, 0x5c, 0xa3, + 0x83, 0x36, 0x24, 0xd4, 0x80, 0x0f, 0xb0, 0x81, 0xb1, + 0xcf, 0x94, 0xeb, + }, + }, + }, + }, + // Key 10 + {"ae45ed5601cec6b8cc05f803935c674ddbe0d75c4c09fd7951fc6b0caec313a8df39970c518bffba5ed68f3f0d7f22a4029d413f1ae07e4ebe9e4177ce23e7f5404b569e4ee1bdcf3c1fb03ef113802d4f855eb9b5134b5a7c8085adcae6fa2fa1417ec3763be171b0c62b760ede23c12ad92b980884c641f5a8fac26bdad4a03381a22fe1b754885094c82506d4019a535a286afeb271bb9ba592de18dcf600c2aeeae56e02f7cf79fc14cf3bdc7cd84febbbf950ca90304b2219a7aa063aefa2c3c1980e560cd64afe779585b6107657b957857efde6010988ab7de417fc88d8f384c4e6e72c3f943e0c31c0c4a5cc36f879d8a3ac9d7d59860eaada6b83bb", + 65537, + "056b04216fe5f354ac77250a4b6b0c8525a85c59b0bd80c56450a22d5f438e596a333aa875e291dd43f48cb88b9d5fc0d499f9fcd1c397f9afc070cd9e398c8d19e61db7c7410a6b2675dfbf5d345b804d201add502d5ce2dfcb091ce9997bbebe57306f383e4d588103f036f7e85d1934d152a323e4a8db451d6f4a5b1b0f102cc150e02feee2b88dea4ad4c1baccb24d84072d14e1d24a6771f7408ee30564fb86d4393a34bcf0b788501d193303f13a2284b001f0f649eaf79328d4ac5c430ab4414920a9460ed1b7bc40ec653e876d09abc509ae45b525190116a0c26101848298509c1c3bf3a483e7274054e15e97075036e989f60932807b5257751e79", + []testEncryptOAEPMessage{ + // Example 10.1 + { + []byte{0x8b, 0xba, 0x6b, 0xf8, 0x2a, 0x6c, 0x0f, 0x86, + 0xd5, 0xf1, 0x75, 0x6e, 0x97, 0x95, 0x68, 0x70, 0xb0, + 0x89, 0x53, 0xb0, 0x6b, 0x4e, 0xb2, 0x05, 0xbc, 0x16, + 0x94, 0xee, + }, + []byte{0x47, 0xe1, 0xab, 0x71, 0x19, 0xfe, 0xe5, 0x6c, + 0x95, 0xee, 0x5e, 0xaa, 0xd8, 0x6f, 0x40, 0xd0, 0xaa, + 0x63, 0xbd, 0x33, + }, + []byte{0x53, 0xea, 0x5d, 0xc0, 0x8c, 0xd2, 0x60, 0xfb, + 0x3b, 0x85, 0x85, 0x67, 0x28, 0x7f, 0xa9, 0x15, 0x52, + 0xc3, 0x0b, 0x2f, 0xeb, 0xfb, 0xa2, 0x13, 0xf0, 0xae, + 0x87, 0x70, 0x2d, 0x06, 0x8d, 0x19, 0xba, 0xb0, 0x7f, + 0xe5, 0x74, 0x52, 0x3d, 0xfb, 0x42, 0x13, 0x9d, 0x68, + 0xc3, 0xc5, 0xaf, 0xee, 0xe0, 0xbf, 0xe4, 0xcb, 0x79, + 0x69, 0xcb, 0xf3, 0x82, 0xb8, 0x04, 0xd6, 0xe6, 0x13, + 0x96, 0x14, 0x4e, 0x2d, 0x0e, 0x60, 0x74, 0x1f, 0x89, + 0x93, 0xc3, 0x01, 0x4b, 0x58, 0xb9, 0xb1, 0x95, 0x7a, + 0x8b, 0xab, 0xcd, 0x23, 0xaf, 0x85, 0x4f, 0x4c, 0x35, + 0x6f, 0xb1, 0x66, 0x2a, 0xa7, 0x2b, 0xfc, 0xc7, 0xe5, + 0x86, 0x55, 0x9d, 0xc4, 0x28, 0x0d, 0x16, 0x0c, 0x12, + 0x67, 0x85, 0xa7, 0x23, 0xeb, 0xee, 0xbe, 0xff, 0x71, + 0xf1, 0x15, 0x94, 0x44, 0x0a, 0xae, 0xf8, 0x7d, 0x10, + 0x79, 0x3a, 0x87, 0x74, 0xa2, 0x39, 0xd4, 0xa0, 0x4c, + 0x87, 0xfe, 0x14, 0x67, 0xb9, 0xda, 0xf8, 0x52, 0x08, + 0xec, 0x6c, 0x72, 0x55, 0x79, 0x4a, 0x96, 0xcc, 0x29, + 0x14, 0x2f, 0x9a, 0x8b, 0xd4, 0x18, 0xe3, 0xc1, 0xfd, + 0x67, 0x34, 0x4b, 0x0c, 0xd0, 0x82, 0x9d, 0xf3, 0xb2, + 0xbe, 0xc6, 0x02, 0x53, 0x19, 0x62, 0x93, 0xc6, 0xb3, + 0x4d, 0x3f, 0x75, 0xd3, 0x2f, 0x21, 0x3d, 0xd4, 0x5c, + 0x62, 0x73, 0xd5, 0x05, 0xad, 0xf4, 0xcc, 0xed, 0x10, + 0x57, 0xcb, 0x75, 0x8f, 0xc2, 0x6a, 0xee, 0xfa, 0x44, + 0x12, 0x55, 0xed, 0x4e, 0x64, 0xc1, 0x99, 0xee, 0x07, + 0x5e, 0x7f, 0x16, 0x64, 0x61, 0x82, 0xfd, 0xb4, 0x64, + 0x73, 0x9b, 0x68, 0xab, 0x5d, 0xaf, 0xf0, 0xe6, 0x3e, + 0x95, 0x52, 0x01, 0x68, 0x24, 0xf0, 0x54, 0xbf, 0x4d, + 0x3c, 0x8c, 0x90, 0xa9, 0x7b, 0xb6, 0xb6, 0x55, 0x32, + 0x84, 0xeb, 0x42, 0x9f, 0xcc, + }, + }, + }, + }, +} diff --git a/libgo/go/crypto/sha1/sha1.go b/libgo/go/crypto/sha1/sha1.go new file mode 100644 index 000000000..8716c3591 --- /dev/null +++ b/libgo/go/crypto/sha1/sha1.go @@ -0,0 +1,114 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This package implements the SHA1 hash algorithm as defined in RFC 3174. +package sha1 + +import ( + "hash" + "os" +) + +// The size of a SHA1 checksum in bytes. +const Size = 20 + +const ( + _Chunk = 64 + _Init0 = 0x67452301 + _Init1 = 0xEFCDAB89 + _Init2 = 0x98BADCFE + _Init3 = 0x10325476 + _Init4 = 0xC3D2E1F0 +) + +// digest represents the partial evaluation of a checksum. +type digest struct { + h [5]uint32 + x [_Chunk]byte + nx int + len uint64 +} + +func (d *digest) Reset() { + d.h[0] = _Init0 + d.h[1] = _Init1 + d.h[2] = _Init2 + d.h[3] = _Init3 + d.h[4] = _Init4 + d.nx = 0 + d.len = 0 +} + +// New returns a new hash.Hash computing the SHA1 checksum. +func New() hash.Hash { + d := new(digest) + d.Reset() + return d +} + +func (d *digest) Size() int { return Size } + +func (d *digest) Write(p []byte) (nn int, err os.Error) { + nn = len(p) + d.len += uint64(nn) + if d.nx > 0 { + n := len(p) + if n > _Chunk-d.nx { + n = _Chunk - d.nx + } + for i := 0; i < n; i++ { + d.x[d.nx+i] = p[i] + } + d.nx += n + if d.nx == _Chunk { + _Block(d, d.x[0:]) + d.nx = 0 + } + p = p[n:] + } + n := _Block(d, p) + p = p[n:] + if len(p) > 0 { + d.nx = copy(d.x[:], p) + } + return +} + +func (d0 *digest) Sum() []byte { + // Make a copy of d0 so that caller can keep writing and summing. + d := new(digest) + *d = *d0 + + // Padding. Add a 1 bit and 0 bits until 56 bytes mod 64. + len := d.len + var tmp [64]byte + tmp[0] = 0x80 + if len%64 < 56 { + d.Write(tmp[0 : 56-len%64]) + } else { + d.Write(tmp[0 : 64+56-len%64]) + } + + // Length in bits. + len <<= 3 + for i := uint(0); i < 8; i++ { + tmp[i] = byte(len >> (56 - 8*i)) + } + d.Write(tmp[0:8]) + + if d.nx != 0 { + panic("d.nx != 0") + } + + p := make([]byte, 20) + j := 0 + for _, s := range d.h { + p[j+0] = byte(s >> 24) + p[j+1] = byte(s >> 16) + p[j+2] = byte(s >> 8) + p[j+3] = byte(s >> 0) + j += 4 + } + return p +} diff --git a/libgo/go/crypto/sha1/sha1_test.go b/libgo/go/crypto/sha1/sha1_test.go new file mode 100644 index 000000000..2712fe35e --- /dev/null +++ b/libgo/go/crypto/sha1/sha1_test.go @@ -0,0 +1,73 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// SHA1 hash algorithm. See RFC 3174. + +package sha1 + +import ( + "fmt" + "io" + "testing" +) + +type sha1Test struct { + out string + in string +} + +var golden = []sha1Test{ + {"da39a3ee5e6b4b0d3255bfef95601890afd80709", ""}, + {"86f7e437faa5a7fce15d1ddcb9eaeaea377667b8", "a"}, + {"da23614e02469a0d7c7bd1bdab5c9c474b1904dc", "ab"}, + {"a9993e364706816aba3e25717850c26c9cd0d89d", "abc"}, + {"81fe8bfe87576c3ecb22426f8e57847382917acf", "abcd"}, + {"03de6c570bfe24bfc328ccd7ca46b76eadaf4334", "abcde"}, + {"1f8ac10f23c5b5bc1167bda84b833e5c057a77d2", "abcdef"}, + {"2fb5e13419fc89246865e7a324f476ec624e8740", "abcdefg"}, + {"425af12a0743502b322e93a015bcf868e324d56a", "abcdefgh"}, + {"c63b19f1e4c8b5f76b25c49b8b87f57d8e4872a1", "abcdefghi"}, + {"d68c19a0a345b7eab78d5e11e991c026ec60db63", "abcdefghij"}, + {"ebf81ddcbe5bf13aaabdc4d65354fdf2044f38a7", "Discard medicine more than two years old."}, + {"e5dea09392dd886ca63531aaa00571dc07554bb6", "He who has a shady past knows that nice guys finish last."}, + {"45988f7234467b94e3e9494434c96ee3609d8f8f", "I wouldn't marry him with a ten foot pole."}, + {"55dee037eb7460d5a692d1ce11330b260e40c988", "Free! Free!/A trip/to Mars/for 900/empty jars/Burma Shave"}, + {"b7bc5fb91080c7de6b582ea281f8a396d7c0aee8", "The days of the digital watch are numbered. -Tom Stoppard"}, + {"c3aed9358f7c77f523afe86135f06b95b3999797", "Nepal premier won't resign."}, + {"6e29d302bf6e3a5e4305ff318d983197d6906bb9", "For every action there is an equal and opposite government program."}, + {"597f6a540010f94c15d71806a99a2c8710e747bd", "His money is twice tainted: 'taint yours and 'taint mine."}, + {"6859733b2590a8a091cecf50086febc5ceef1e80", "There is no reason for any individual to have a computer in their home. -Ken Olsen, 1977"}, + {"514b2630ec089b8aee18795fc0cf1f4860cdacad", "It's a tiny change to the code and not completely disgusting. - Bob Manchek"}, + {"c5ca0d4a7b6676fc7aa72caa41cc3d5df567ed69", "size: a.out: bad magic"}, + {"74c51fa9a04eadc8c1bbeaa7fc442f834b90a00a", "The major problem is with sendmail. -Mark Horton"}, + {"0b4c4ce5f52c3ad2821852a8dc00217fa18b8b66", "Give me a rock, paper and scissors and I will move the world. CCFestoon"}, + {"3ae7937dd790315beb0f48330e8642237c61550a", "If the enemy is within range, then so are you."}, + {"410a2b296df92b9a47412b13281df8f830a9f44b", "It's well we cannot hear the screams/That we create in others' dreams."}, + {"841e7c85ca1adcddbdd0187f1289acb5c642f7f5", "You remind me of a TV show, but that's all right: I watch it anyway."}, + {"163173b825d03b952601376b25212df66763e1db", "C is as portable as Stonehedge!!"}, + {"32b0377f2687eb88e22106f133c586ab314d5279", "Even if I could be Shakespeare, I think I should still choose to be Faraday. - A. Huxley"}, + {"0885aaf99b569542fd165fa44e322718f4a984e0", "The fugacity of a constituent in a mixture of gases at a given temperature is proportional to its mole fraction. Lewis-Randall Rule"}, + {"6627d6904d71420b0bf3886ab629623538689f45", "How can you write a big system without C++? -Paul Glick"}, +} + +func TestGolden(t *testing.T) { + for i := 0; i < len(golden); i++ { + g := golden[i] + c := New() + for j := 0; j < 3; j++ { + if j < 2 { + io.WriteString(c, g.in) + } else { + io.WriteString(c, g.in[0:len(g.in)/2]) + c.Sum() + io.WriteString(c, g.in[len(g.in)/2:]) + } + s := fmt.Sprintf("%x", c.Sum()) + if s != g.out { + t.Fatalf("sha1[%d](%s) = %s want %s", j, g.in, s, g.out) + } + c.Reset() + } + } +} diff --git a/libgo/go/crypto/sha1/sha1block.go b/libgo/go/crypto/sha1/sha1block.go new file mode 100644 index 000000000..b5d32af70 --- /dev/null +++ b/libgo/go/crypto/sha1/sha1block.go @@ -0,0 +1,81 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// SHA1 block step. +// In its own file so that a faster assembly or C version +// can be substituted easily. + +package sha1 + +const ( + _K0 = 0x5A827999 + _K1 = 0x6ED9EBA1 + _K2 = 0x8F1BBCDC + _K3 = 0xCA62C1D6 +) + +func _Block(dig *digest, p []byte) int { + var w [80]uint32 + + n := 0 + h0, h1, h2, h3, h4 := dig.h[0], dig.h[1], dig.h[2], dig.h[3], dig.h[4] + for len(p) >= _Chunk { + // Can interlace the computation of w with the + // rounds below if needed for speed. + for i := 0; i < 16; i++ { + j := i * 4 + w[i] = uint32(p[j])<<24 | uint32(p[j+1])<<16 | uint32(p[j+2])<<8 | uint32(p[j+3]) + } + for i := 16; i < 80; i++ { + tmp := w[i-3] ^ w[i-8] ^ w[i-14] ^ w[i-16] + w[i] = tmp<<1 | tmp>>(32-1) + } + + a, b, c, d, e := h0, h1, h2, h3, h4 + + // Each of the four 20-iteration rounds + // differs only in the computation of f and + // the choice of K (_K0, _K1, etc). + for i := 0; i < 20; i++ { + f := b&c | (^b)&d + a5 := a<<5 | a>>(32-5) + b30 := b<<30 | b>>(32-30) + t := a5 + f + e + w[i] + _K0 + a, b, c, d, e = t, a, b30, c, d + } + for i := 20; i < 40; i++ { + f := b ^ c ^ d + a5 := a<<5 | a>>(32-5) + b30 := b<<30 | b>>(32-30) + t := a5 + f + e + w[i] + _K1 + a, b, c, d, e = t, a, b30, c, d + } + for i := 40; i < 60; i++ { + f := b&c | b&d | c&d + a5 := a<<5 | a>>(32-5) + b30 := b<<30 | b>>(32-30) + t := a5 + f + e + w[i] + _K2 + a, b, c, d, e = t, a, b30, c, d + } + for i := 60; i < 80; i++ { + f := b ^ c ^ d + a5 := a<<5 | a>>(32-5) + b30 := b<<30 | b>>(32-30) + t := a5 + f + e + w[i] + _K3 + a, b, c, d, e = t, a, b30, c, d + } + + h0 += a + h1 += b + h2 += c + h3 += d + h4 += e + + p = p[_Chunk:] + n += _Chunk + } + + dig.h[0], dig.h[1], dig.h[2], dig.h[3], dig.h[4] = h0, h1, h2, h3, h4 + return n +} diff --git a/libgo/go/crypto/sha256/sha256.go b/libgo/go/crypto/sha256/sha256.go new file mode 100644 index 000000000..57a8ffa0d --- /dev/null +++ b/libgo/go/crypto/sha256/sha256.go @@ -0,0 +1,159 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This package implements the SHA224 and SHA256 hash algorithms as defined in FIPS 180-2. +package sha256 + +import ( + "hash" + "os" +) + +// The size of a SHA256 checksum in bytes. +const Size = 32 + +// The size of a SHA224 checksum in bytes. +const Size224 = 28 + +const ( + _Chunk = 64 + _Init0 = 0x6A09E667 + _Init1 = 0xBB67AE85 + _Init2 = 0x3C6EF372 + _Init3 = 0xA54FF53A + _Init4 = 0x510E527F + _Init5 = 0x9B05688C + _Init6 = 0x1F83D9AB + _Init7 = 0x5BE0CD19 + _Init0_224 = 0xC1059ED8 + _Init1_224 = 0x367CD507 + _Init2_224 = 0x3070DD17 + _Init3_224 = 0xF70E5939 + _Init4_224 = 0xFFC00B31 + _Init5_224 = 0x68581511 + _Init6_224 = 0x64F98FA7 + _Init7_224 = 0xBEFA4FA4 +) + +// digest represents the partial evaluation of a checksum. +type digest struct { + h [8]uint32 + x [_Chunk]byte + nx int + len uint64 + is224 bool // mark if this digest is SHA-224 +} + +func (d *digest) Reset() { + if !d.is224 { + d.h[0] = _Init0 + d.h[1] = _Init1 + d.h[2] = _Init2 + d.h[3] = _Init3 + d.h[4] = _Init4 + d.h[5] = _Init5 + d.h[6] = _Init6 + d.h[7] = _Init7 + } else { + d.h[0] = _Init0_224 + d.h[1] = _Init1_224 + d.h[2] = _Init2_224 + d.h[3] = _Init3_224 + d.h[4] = _Init4_224 + d.h[5] = _Init5_224 + d.h[6] = _Init6_224 + d.h[7] = _Init7_224 + } + d.nx = 0 + d.len = 0 +} + +// New returns a new hash.Hash computing the SHA256 checksum. +func New() hash.Hash { + d := new(digest) + d.Reset() + return d +} + +// New224 returns a new hash.Hash computing the SHA224 checksum. +func New224() hash.Hash { + d := new(digest) + d.is224 = true + d.Reset() + return d +} + +func (d *digest) Size() int { + if !d.is224 { + return Size + } + return Size224 +} + +func (d *digest) Write(p []byte) (nn int, err os.Error) { + nn = len(p) + d.len += uint64(nn) + if d.nx > 0 { + n := len(p) + if n > _Chunk-d.nx { + n = _Chunk - d.nx + } + for i := 0; i < n; i++ { + d.x[d.nx+i] = p[i] + } + d.nx += n + if d.nx == _Chunk { + _Block(d, d.x[0:]) + d.nx = 0 + } + p = p[n:] + } + n := _Block(d, p) + p = p[n:] + if len(p) > 0 { + d.nx = copy(d.x[:], p) + } + return +} + +func (d0 *digest) Sum() []byte { + // Make a copy of d0 so that caller can keep writing and summing. + d := new(digest) + *d = *d0 + + // Padding. Add a 1 bit and 0 bits until 56 bytes mod 64. + len := d.len + var tmp [64]byte + tmp[0] = 0x80 + if len%64 < 56 { + d.Write(tmp[0 : 56-len%64]) + } else { + d.Write(tmp[0 : 64+56-len%64]) + } + + // Length in bits. + len <<= 3 + for i := uint(0); i < 8; i++ { + tmp[i] = byte(len >> (56 - 8*i)) + } + d.Write(tmp[0:8]) + + if d.nx != 0 { + panic("d.nx != 0") + } + + p := make([]byte, 32) + j := 0 + for _, s := range d.h { + p[j+0] = byte(s >> 24) + p[j+1] = byte(s >> 16) + p[j+2] = byte(s >> 8) + p[j+3] = byte(s >> 0) + j += 4 + } + if d.is224 { + return p[0:28] + } + return p +} diff --git a/libgo/go/crypto/sha256/sha256_test.go b/libgo/go/crypto/sha256/sha256_test.go new file mode 100644 index 000000000..42a3fa7a0 --- /dev/null +++ b/libgo/go/crypto/sha256/sha256_test.go @@ -0,0 +1,125 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// SHA256 hash algorithm. See FIPS 180-2. + +package sha256 + +import ( + "fmt" + "io" + "testing" +) + +type sha256Test struct { + out string + in string +} + +var golden = []sha256Test{ + {"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", ""}, + {"ca978112ca1bbdcafac231b39a23dc4da786eff8147c4e72b9807785afee48bb", "a"}, + {"fb8e20fc2e4c3f248c60c39bd652f3c1347298bb977b8b4d5903b85055620603", "ab"}, + {"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", "abc"}, + {"88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589", "abcd"}, + {"36bbe50ed96841d10443bcb670d6554f0a34b761be67ec9c4a8ad2c0c44ca42c", "abcde"}, + {"bef57ec7f53a6d40beb640a780a639c83bc29ac8a9816f1fc6c5c6dcd93c4721", "abcdef"}, + {"7d1a54127b222502f5b79b5fb0803061152a44f92b37e23c6527baf665d4da9a", "abcdefg"}, + {"9c56cc51b374c3ba189210d5b6d4bf57790d351c96c47c02190ecf1e430635ab", "abcdefgh"}, + {"19cc02f26df43cc571bc9ed7b0c4d29224a3ec229529221725ef76d021c8326f", "abcdefghi"}, + {"72399361da6a7754fec986dca5b7cbaf1c810a28ded4abaf56b2106d06cb78b0", "abcdefghij"}, + {"a144061c271f152da4d151034508fed1c138b8c976339de229c3bb6d4bbb4fce", "Discard medicine more than two years old."}, + {"6dae5caa713a10ad04b46028bf6dad68837c581616a1589a265a11288d4bb5c4", "He who has a shady past knows that nice guys finish last."}, + {"ae7a702a9509039ddbf29f0765e70d0001177914b86459284dab8b348c2dce3f", "I wouldn't marry him with a ten foot pole."}, + {"6748450b01c568586715291dfa3ee018da07d36bb7ea6f180c1af6270215c64f", "Free! Free!/A trip/to Mars/for 900/empty jars/Burma Shave"}, + {"14b82014ad2b11f661b5ae6a99b75105c2ffac278cd071cd6c05832793635774", "The days of the digital watch are numbered. -Tom Stoppard"}, + {"7102cfd76e2e324889eece5d6c41921b1e142a4ac5a2692be78803097f6a48d8", "Nepal premier won't resign."}, + {"23b1018cd81db1d67983c5f7417c44da9deb582459e378d7a068552ea649dc9f", "For every action there is an equal and opposite government program."}, + {"8001f190dfb527261c4cfcab70c98e8097a7a1922129bc4096950e57c7999a5a", "His money is twice tainted: 'taint yours and 'taint mine."}, + {"8c87deb65505c3993eb24b7a150c4155e82eee6960cf0c3a8114ff736d69cad5", "There is no reason for any individual to have a computer in their home. -Ken Olsen, 1977"}, + {"bfb0a67a19cdec3646498b2e0f751bddc41bba4b7f30081b0b932aad214d16d7", "It's a tiny change to the code and not completely disgusting. - Bob Manchek"}, + {"7f9a0b9bf56332e19f5a0ec1ad9c1425a153da1c624868fda44561d6b74daf36", "size: a.out: bad magic"}, + {"b13f81b8aad9e3666879af19886140904f7f429ef083286195982a7588858cfc", "The major problem is with sendmail. -Mark Horton"}, + {"b26c38d61519e894480c70c8374ea35aa0ad05b2ae3d6674eec5f52a69305ed4", "Give me a rock, paper and scissors and I will move the world. CCFestoon"}, + {"049d5e26d4f10222cd841a119e38bd8d2e0d1129728688449575d4ff42b842c1", "If the enemy is within range, then so are you."}, + {"0e116838e3cc1c1a14cd045397e29b4d087aa11b0853fc69ec82e90330d60949", "It's well we cannot hear the screams/That we create in others' dreams."}, + {"4f7d8eb5bcf11de2a56b971021a444aa4eafd6ecd0f307b5109e4e776cd0fe46", "You remind me of a TV show, but that's all right: I watch it anyway."}, + {"61c0cc4c4bd8406d5120b3fb4ebc31ce87667c162f29468b3c779675a85aebce", "C is as portable as Stonehedge!!"}, + {"1fb2eb3688093c4a3f80cd87a5547e2ce940a4f923243a79a2a1e242220693ac", "Even if I could be Shakespeare, I think I should still choose to be Faraday. - A. Huxley"}, + {"395585ce30617b62c80b93e8208ce866d4edc811a177fdb4b82d3911d8696423", "The fugacity of a constituent in a mixture of gases at a given temperature is proportional to its mole fraction. Lewis-Randall Rule"}, + {"4f9b189a13d030838269dce846b16a1ce9ce81fe63e65de2f636863336a98fe6", "How can you write a big system without C++? -Paul Glick"}, +} + +var golden224 = []sha256Test{ + {"d14a028c2a3a2bc9476102bb288234c415a2b01f828ea62ac5b3e42f", ""}, + {"abd37534c7d9a2efb9465de931cd7055ffdb8879563ae98078d6d6d5", "a"}, + {"db3cda86d4429a1d39c148989566b38f7bda0156296bd364ba2f878b", "ab"}, + {"23097d223405d8228642a477bda255b32aadbce4bda0b3f7e36c9da7", "abc"}, + {"a76654d8e3550e9a2d67a0eeb6c67b220e5885eddd3fde135806e601", "abcd"}, + {"bdd03d560993e675516ba5a50638b6531ac2ac3d5847c61916cfced6", "abcde"}, + {"7043631cb415556a275a4ebecb802c74ee9f6153908e1792a90b6a98", "abcdef"}, + {"d1884e711701ad81abe0c77a3b0ea12e19ba9af64077286c72fc602d", "abcdefg"}, + {"17eb7d40f0356f8598e89eafad5f6c759b1f822975d9c9b737c8a517", "abcdefgh"}, + {"aeb35915346c584db820d2de7af3929ffafef9222a9bcb26516c7334", "abcdefghi"}, + {"d35e1e5af29ddb0d7e154357df4ad9842afee527c689ee547f753188", "abcdefghij"}, + {"19297f1cef7ddc8a7e947f5c5a341e10f7245045e425db67043988d7", "Discard medicine more than two years old."}, + {"0f10c2eb436251f777fbbd125e260d36aecf180411726c7c885f599a", "He who has a shady past knows that nice guys finish last."}, + {"4d1842104919f314cad8a3cd20b3cba7e8ed3e7abed62b57441358f6", "I wouldn't marry him with a ten foot pole."}, + {"a8ba85c6fe0c48fbffc72bbb2f03fcdbc87ae2dc7a56804d1590fb3b", "Free! Free!/A trip/to Mars/for 900/empty jars/Burma Shave"}, + {"5543fbab26e67e8885b1a852d567d1cb8b9bfe42e0899584c50449a9", "The days of the digital watch are numbered. -Tom Stoppard"}, + {"65ca107390f5da9efa05d28e57b221657edc7e43a9a18fb15b053ddb", "Nepal premier won't resign."}, + {"84953962be366305a9cc9b5cd16ed019edc37ac96c0deb3e12cca116", "For every action there is an equal and opposite government program."}, + {"35a189ce987151dfd00b3577583cc6a74b9869eecf894459cb52038d", "His money is twice tainted: 'taint yours and 'taint mine."}, + {"2fc333713983edfd4ef2c0da6fb6d6415afb94987c91e4069eb063e6", "There is no reason for any individual to have a computer in their home. -Ken Olsen, 1977"}, + {"cbe32d38d577a1b355960a4bc3c659c2dc4670859a19777a875842c4", "It's a tiny change to the code and not completely disgusting. - Bob Manchek"}, + {"a2dc118ce959e027576413a7b440c875cdc8d40df9141d6ef78a57e1", "size: a.out: bad magic"}, + {"d10787e24052bcff26dc484787a54ed819e4e4511c54890ee977bf81", "The major problem is with sendmail. -Mark Horton"}, + {"62efcf16ab8a893acdf2f348aaf06b63039ff1bf55508c830532c9fb", "Give me a rock, paper and scissors and I will move the world. CCFestoon"}, + {"3e9b7e4613c59f58665104c5fa86c272db5d3a2ff30df5bb194a5c99", "If the enemy is within range, then so are you."}, + {"5999c208b8bdf6d471bb7c359ac5b829e73a8211dff686143a4e7f18", "It's well we cannot hear the screams/That we create in others' dreams."}, + {"3b2d67ff54eabc4ef737b14edf87c64280ef582bcdf2a6d56908b405", "You remind me of a TV show, but that's all right: I watch it anyway."}, + {"d0733595d20e4d3d6b5c565a445814d1bbb2fd08b9a3b8ffb97930c6", "C is as portable as Stonehedge!!"}, + {"43fb8aeed8a833175c9295c1165415f98c866ef08a4922959d673507", "Even if I could be Shakespeare, I think I should still choose to be Faraday. - A. Huxley"}, + {"ec18e66e93afc4fb1604bc2baedbfd20b44c43d76e65c0996d7851c6", "The fugacity of a constituent in a mixture of gases at a given temperature is proportional to its mole fraction. Lewis-Randall Rule"}, + {"86ed2eaa9c75ba98396e5c9fb2f679ecf0ea2ed1e0ee9ceecb4a9332", "How can you write a big system without C++? -Paul Glick"}, +} + +func TestGolden(t *testing.T) { + for i := 0; i < len(golden); i++ { + g := golden[i] + c := New() + for j := 0; j < 3; j++ { + if j < 2 { + io.WriteString(c, g.in) + } else { + io.WriteString(c, g.in[0:len(g.in)/2]) + c.Sum() + io.WriteString(c, g.in[len(g.in)/2:]) + } + s := fmt.Sprintf("%x", c.Sum()) + if s != g.out { + t.Fatalf("sha256[%d](%s) = %s want %s", j, g.in, s, g.out) + } + c.Reset() + } + } + for i := 0; i < len(golden224); i++ { + g := golden224[i] + c := New224() + for j := 0; j < 3; j++ { + if j < 2 { + io.WriteString(c, g.in) + } else { + io.WriteString(c, g.in[0:len(g.in)/2]) + c.Sum() + io.WriteString(c, g.in[len(g.in)/2:]) + } + s := fmt.Sprintf("%x", c.Sum()) + if s != g.out { + t.Fatalf("sha224[%d](%s) = %s want %s", j, g.in, s, g.out) + } + c.Reset() + } + } +} diff --git a/libgo/go/crypto/sha256/sha256block.go b/libgo/go/crypto/sha256/sha256block.go new file mode 100644 index 000000000..7b0f55444 --- /dev/null +++ b/libgo/go/crypto/sha256/sha256block.go @@ -0,0 +1,129 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// SHA256 block step. +// In its own file so that a faster assembly or C version +// can be substituted easily. + +package sha256 + +var _K = []uint32{ + 0x428a2f98, + 0x71374491, + 0xb5c0fbcf, + 0xe9b5dba5, + 0x3956c25b, + 0x59f111f1, + 0x923f82a4, + 0xab1c5ed5, + 0xd807aa98, + 0x12835b01, + 0x243185be, + 0x550c7dc3, + 0x72be5d74, + 0x80deb1fe, + 0x9bdc06a7, + 0xc19bf174, + 0xe49b69c1, + 0xefbe4786, + 0x0fc19dc6, + 0x240ca1cc, + 0x2de92c6f, + 0x4a7484aa, + 0x5cb0a9dc, + 0x76f988da, + 0x983e5152, + 0xa831c66d, + 0xb00327c8, + 0xbf597fc7, + 0xc6e00bf3, + 0xd5a79147, + 0x06ca6351, + 0x14292967, + 0x27b70a85, + 0x2e1b2138, + 0x4d2c6dfc, + 0x53380d13, + 0x650a7354, + 0x766a0abb, + 0x81c2c92e, + 0x92722c85, + 0xa2bfe8a1, + 0xa81a664b, + 0xc24b8b70, + 0xc76c51a3, + 0xd192e819, + 0xd6990624, + 0xf40e3585, + 0x106aa070, + 0x19a4c116, + 0x1e376c08, + 0x2748774c, + 0x34b0bcb5, + 0x391c0cb3, + 0x4ed8aa4a, + 0x5b9cca4f, + 0x682e6ff3, + 0x748f82ee, + 0x78a5636f, + 0x84c87814, + 0x8cc70208, + 0x90befffa, + 0xa4506ceb, + 0xbef9a3f7, + 0xc67178f2, +} + +func _Block(dig *digest, p []byte) int { + var w [64]uint32 + n := 0 + h0, h1, h2, h3, h4, h5, h6, h7 := dig.h[0], dig.h[1], dig.h[2], dig.h[3], dig.h[4], dig.h[5], dig.h[6], dig.h[7] + for len(p) >= _Chunk { + // Can interlace the computation of w with the + // rounds below if needed for speed. + for i := 0; i < 16; i++ { + j := i * 4 + w[i] = uint32(p[j])<<24 | uint32(p[j+1])<<16 | uint32(p[j+2])<<8 | uint32(p[j+3]) + } + for i := 16; i < 64; i++ { + t1 := (w[i-2]>>17 | w[i-2]<<(32-17)) ^ (w[i-2]>>19 | w[i-2]<<(32-19)) ^ (w[i-2] >> 10) + + t2 := (w[i-15]>>7 | w[i-15]<<(32-7)) ^ (w[i-15]>>18 | w[i-15]<<(32-18)) ^ (w[i-15] >> 3) + + w[i] = t1 + w[i-7] + t2 + w[i-16] + } + + a, b, c, d, e, f, g, h := h0, h1, h2, h3, h4, h5, h6, h7 + + for i := 0; i < 64; i++ { + t1 := h + ((e>>6 | e<<(32-6)) ^ (e>>11 | e<<(32-11)) ^ (e>>25 | e<<(32-25))) + ((e & f) ^ (^e & g)) + _K[i] + w[i] + + t2 := ((a>>2 | a<<(32-2)) ^ (a>>13 | a<<(32-13)) ^ (a>>22 | a<<(32-22))) + ((a & b) ^ (a & c) ^ (b & c)) + + h = g + g = f + f = e + e = d + t1 + d = c + c = b + b = a + a = t1 + t2 + } + + h0 += a + h1 += b + h2 += c + h3 += d + h4 += e + h5 += f + h6 += g + h7 += h + + p = p[_Chunk:] + n += _Chunk + } + + dig.h[0], dig.h[1], dig.h[2], dig.h[3], dig.h[4], dig.h[5], dig.h[6], dig.h[7] = h0, h1, h2, h3, h4, h5, h6, h7 + return n +} diff --git a/libgo/go/crypto/sha512/sha512.go b/libgo/go/crypto/sha512/sha512.go new file mode 100644 index 000000000..c3cda97d9 --- /dev/null +++ b/libgo/go/crypto/sha512/sha512.go @@ -0,0 +1,163 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This package implements the SHA384 and SHA512 hash algorithms as defined in FIPS 180-2. +package sha512 + +import ( + "hash" + "os" +) + +// The size of a SHA512 checksum in bytes. +const Size = 64 + +// The size of a SHA384 checksum in bytes. +const Size384 = 48 + +const ( + _Chunk = 128 + _Init0 = 0x6a09e667f3bcc908 + _Init1 = 0xbb67ae8584caa73b + _Init2 = 0x3c6ef372fe94f82b + _Init3 = 0xa54ff53a5f1d36f1 + _Init4 = 0x510e527fade682d1 + _Init5 = 0x9b05688c2b3e6c1f + _Init6 = 0x1f83d9abfb41bd6b + _Init7 = 0x5be0cd19137e2179 + _Init0_384 = 0xcbbb9d5dc1059ed8 + _Init1_384 = 0x629a292a367cd507 + _Init2_384 = 0x9159015a3070dd17 + _Init3_384 = 0x152fecd8f70e5939 + _Init4_384 = 0x67332667ffc00b31 + _Init5_384 = 0x8eb44a8768581511 + _Init6_384 = 0xdb0c2e0d64f98fa7 + _Init7_384 = 0x47b5481dbefa4fa4 +) + +// digest represents the partial evaluation of a checksum. +type digest struct { + h [8]uint64 + x [_Chunk]byte + nx int + len uint64 + is384 bool // mark if this digest is SHA-384 +} + +func (d *digest) Reset() { + if !d.is384 { + d.h[0] = _Init0 + d.h[1] = _Init1 + d.h[2] = _Init2 + d.h[3] = _Init3 + d.h[4] = _Init4 + d.h[5] = _Init5 + d.h[6] = _Init6 + d.h[7] = _Init7 + } else { + d.h[0] = _Init0_384 + d.h[1] = _Init1_384 + d.h[2] = _Init2_384 + d.h[3] = _Init3_384 + d.h[4] = _Init4_384 + d.h[5] = _Init5_384 + d.h[6] = _Init6_384 + d.h[7] = _Init7_384 + } + d.nx = 0 + d.len = 0 +} + +// New returns a new hash.Hash computing the SHA512 checksum. +func New() hash.Hash { + d := new(digest) + d.Reset() + return d +} + +// New384 returns a new hash.Hash computing the SHA384 checksum. +func New384() hash.Hash { + d := new(digest) + d.is384 = true + d.Reset() + return d +} + +func (d *digest) Size() int { + if !d.is384 { + return Size + } + return Size384 +} + +func (d *digest) Write(p []byte) (nn int, err os.Error) { + nn = len(p) + d.len += uint64(nn) + if d.nx > 0 { + n := len(p) + if n > _Chunk-d.nx { + n = _Chunk - d.nx + } + for i := 0; i < n; i++ { + d.x[d.nx+i] = p[i] + } + d.nx += n + if d.nx == _Chunk { + _Block(d, d.x[0:]) + d.nx = 0 + } + p = p[n:] + } + n := _Block(d, p) + p = p[n:] + if len(p) > 0 { + d.nx = copy(d.x[:], p) + } + return +} + +func (d0 *digest) Sum() []byte { + // Make a copy of d0 so that caller can keep writing and summing. + d := new(digest) + *d = *d0 + + // Padding. Add a 1 bit and 0 bits until 112 bytes mod 128. + len := d.len + var tmp [128]byte + tmp[0] = 0x80 + if len%128 < 112 { + d.Write(tmp[0 : 112-len%128]) + } else { + d.Write(tmp[0 : 128+112-len%128]) + } + + // Length in bits. + len <<= 3 + for i := uint(0); i < 16; i++ { + tmp[i] = byte(len >> (120 - 8*i)) + } + d.Write(tmp[0:16]) + + if d.nx != 0 { + panic("d.nx != 0") + } + + p := make([]byte, 64) + j := 0 + for _, s := range d.h { + p[j+0] = byte(s >> 56) + p[j+1] = byte(s >> 48) + p[j+2] = byte(s >> 40) + p[j+3] = byte(s >> 32) + p[j+4] = byte(s >> 24) + p[j+5] = byte(s >> 16) + p[j+6] = byte(s >> 8) + p[j+7] = byte(s >> 0) + j += 8 + } + if d.is384 { + return p[0:48] + } + return p +} diff --git a/libgo/go/crypto/sha512/sha512_test.go b/libgo/go/crypto/sha512/sha512_test.go new file mode 100644 index 000000000..dd116dc17 --- /dev/null +++ b/libgo/go/crypto/sha512/sha512_test.go @@ -0,0 +1,125 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// SHA512 hash algorithm. See FIPS 180-2. + +package sha512 + +import ( + "fmt" + "io" + "testing" +) + +type sha512Test struct { + out string + in string +} + +var golden = []sha512Test{ + {"cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e", ""}, + {"1f40fc92da241694750979ee6cf582f2d5d7d28e18335de05abc54d0560e0f5302860c652bf08d560252aa5e74210546f369fbbbce8c12cfc7957b2652fe9a75", "a"}, + {"2d408a0717ec188158278a796c689044361dc6fdde28d6f04973b80896e1823975cdbf12eb63f9e0591328ee235d80e9b5bf1aa6a44f4617ff3caf6400eb172d", "ab"}, + {"ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f", "abc"}, + {"d8022f2060ad6efd297ab73dcc5355c9b214054b0d1776a136a669d26a7d3b14f73aa0d0ebff19ee333368f0164b6419a96da49e3e481753e7e96b716bdccb6f", "abcd"}, + {"878ae65a92e86cac011a570d4c30a7eaec442b85ce8eca0c2952b5e3cc0628c2e79d889ad4d5c7c626986d452dd86374b6ffaa7cd8b67665bef2289a5c70b0a1", "abcde"}, + {"e32ef19623e8ed9d267f657a81944b3d07adbb768518068e88435745564e8d4150a0a703be2a7d88b61e3d390c2bb97e2d4c311fdc69d6b1267f05f59aa920e7", "abcdef"}, + {"d716a4188569b68ab1b6dfac178e570114cdf0ea3a1cc0e31486c3e41241bc6a76424e8c37ab26f096fc85ef9886c8cb634187f4fddff645fb099f1ff54c6b8c", "abcdefg"}, + {"a3a8c81bc97c2560010d7389bc88aac974a104e0e2381220c6e084c4dccd1d2d17d4f86db31c2a851dc80e6681d74733c55dcd03dd96f6062cdda12a291ae6ce", "abcdefgh"}, + {"f22d51d25292ca1d0f68f69aedc7897019308cc9db46efb75a03dd494fc7f126c010e8ade6a00a0c1a5f1b75d81e0ed5a93ce98dc9b833db7839247b1d9c24fe", "abcdefghi"}, + {"ef6b97321f34b1fea2169a7db9e1960b471aa13302a988087357c520be957ca119c3ba68e6b4982c019ec89de3865ccf6a3cda1fe11e59f98d99f1502c8b9745", "abcdefghij"}, + {"2210d99af9c8bdecda1b4beff822136753d8342505ddce37f1314e2cdbb488c6016bdaa9bd2ffa513dd5de2e4b50f031393d8ab61f773b0e0130d7381e0f8a1d", "Discard medicine more than two years old."}, + {"a687a8985b4d8d0a24f115fe272255c6afaf3909225838546159c1ed685c211a203796ae8ecc4c81a5b6315919b3a64f10713da07e341fcdbb08541bf03066ce", "He who has a shady past knows that nice guys finish last."}, + {"8ddb0392e818b7d585ab22769a50df660d9f6d559cca3afc5691b8ca91b8451374e42bcdabd64589ed7c91d85f626596228a5c8572677eb98bc6b624befb7af8", "I wouldn't marry him with a ten foot pole."}, + {"26ed8f6ca7f8d44b6a8a54ae39640fa8ad5c673f70ee9ce074ba4ef0d483eea00bab2f61d8695d6b34df9c6c48ae36246362200ed820448bdc03a720366a87c6", "Free! Free!/A trip/to Mars/for 900/empty jars/Burma Shave"}, + {"e5a14bf044be69615aade89afcf1ab0389d5fc302a884d403579d1386a2400c089b0dbb387ed0f463f9ee342f8244d5a38cfbc0e819da9529fbff78368c9a982", "The days of the digital watch are numbered. -Tom Stoppard"}, + {"420a1faa48919e14651bed45725abe0f7a58e0f099424c4e5a49194946e38b46c1f8034b18ef169b2e31050d1648e0b982386595f7df47da4b6fd18e55333015", "Nepal premier won't resign."}, + {"d926a863beadb20134db07683535c72007b0e695045876254f341ddcccde132a908c5af57baa6a6a9c63e6649bba0c213dc05fadcf9abccea09f23dcfb637fbe", "For every action there is an equal and opposite government program."}, + {"9a98dd9bb67d0da7bf83da5313dff4fd60a4bac0094f1b05633690ffa7f6d61de9a1d4f8617937d560833a9aaa9ccafe3fd24db418d0e728833545cadd3ad92d", "His money is twice tainted: 'taint yours and 'taint mine."}, + {"d7fde2d2351efade52f4211d3746a0780a26eec3df9b2ed575368a8a1c09ec452402293a8ea4eceb5a4f60064ea29b13cdd86918cd7a4faf366160b009804107", "There is no reason for any individual to have a computer in their home. -Ken Olsen, 1977"}, + {"b0f35ffa2697359c33a56f5c0cf715c7aeed96da9905ca2698acadb08fbc9e669bf566b6bd5d61a3e86dc22999bcc9f2224e33d1d4f32a228cf9d0349e2db518", "It's a tiny change to the code and not completely disgusting. - Bob Manchek"}, + {"3d2e5f91778c9e66f7e061293aaa8a8fc742dd3b2e4f483772464b1144189b49273e610e5cccd7a81a19ca1fa70f16b10f1a100a4d8c1372336be8484c64b311", "size: a.out: bad magic"}, + {"b2f68ff58ac015efb1c94c908b0d8c2bf06f491e4de8e6302c49016f7f8a33eac3e959856c7fddbc464de618701338a4b46f76dbfaf9a1e5262b5f40639771c7", "The major problem is with sendmail. -Mark Horton"}, + {"d8c92db5fdf52cf8215e4df3b4909d29203ff4d00e9ad0b64a6a4e04dec5e74f62e7c35c7fb881bd5de95442123df8f57a489b0ae616bd326f84d10021121c57", "Give me a rock, paper and scissors and I will move the world. CCFestoon"}, + {"19a9f8dc0a233e464e8566ad3ca9b91e459a7b8c4780985b015776e1bf239a19bc233d0556343e2b0a9bc220900b4ebf4f8bdf89ff8efeaf79602d6849e6f72e", "If the enemy is within range, then so are you."}, + {"00b4c41f307bde87301cdc5b5ab1ae9a592e8ecbb2021dd7bc4b34e2ace60741cc362560bec566ba35178595a91932b8d5357e2c9cec92d393b0fa7831852476", "It's well we cannot hear the screams/That we create in others' dreams."}, + {"91eccc3d5375fd026e4d6787874b1dce201cecd8a27dbded5065728cb2d09c58a3d467bb1faf353bf7ba567e005245d5321b55bc344f7c07b91cb6f26c959be7", "You remind me of a TV show, but that's all right: I watch it anyway."}, + {"fabbbe22180f1f137cfdc9556d2570e775d1ae02a597ded43a72a40f9b485d500043b7be128fb9fcd982b83159a0d99aa855a9e7cc4240c00dc01a9bdf8218d7", "C is as portable as Stonehedge!!"}, + {"2ecdec235c1fa4fc2a154d8fba1dddb8a72a1ad73838b51d792331d143f8b96a9f6fcb0f34d7caa351fe6d88771c4f105040e0392f06e0621689d33b2f3ba92e", "Even if I could be Shakespeare, I think I should still choose to be Faraday. - A. Huxley"}, + {"7ad681f6f96f82f7abfa7ecc0334e8fa16d3dc1cdc45b60b7af43fe4075d2357c0c1d60e98350f1afb1f2fe7a4d7cd2ad55b88e458e06b73c40b437331f5dab4", "The fugacity of a constituent in a mixture of gases at a given temperature is proportional to its mole fraction. Lewis-Randall Rule"}, + {"833f9248ab4a3b9e5131f745fda1ffd2dd435b30e965957e78291c7ab73605fd1912b0794e5c233ab0a12d205a39778d19b83515d6a47003f19cdee51d98c7e0", "How can you write a big system without C++? -Paul Glick"}, +} + +var golden384 = []sha512Test{ + {"38b060a751ac96384cd9327eb1b1e36a21fdb71114be07434c0cc7bf63f6e1da274edebfe76f65fbd51ad2f14898b95b", ""}, + {"54a59b9f22b0b80880d8427e548b7c23abd873486e1f035dce9cd697e85175033caa88e6d57bc35efae0b5afd3145f31", "a"}, + {"c7be03ba5bcaa384727076db0018e99248e1a6e8bd1b9ef58a9ec9dd4eeebb3f48b836201221175befa74ddc3d35afdd", "ab"}, + {"cb00753f45a35e8bb5a03d699ac65007272c32ab0eded1631a8b605a43ff5bed8086072ba1e7cc2358baeca134c825a7", "abc"}, + {"1165b3406ff0b52a3d24721f785462ca2276c9f454a116c2b2ba20171a7905ea5a026682eb659c4d5f115c363aa3c79b", "abcd"}, + {"4c525cbeac729eaf4b4665815bc5db0c84fe6300068a727cf74e2813521565abc0ec57a37ee4d8be89d097c0d2ad52f0", "abcde"}, + {"c6a4c65b227e7387b9c3e839d44869c4cfca3ef583dea64117859b808c1e3d8ae689e1e314eeef52a6ffe22681aa11f5", "abcdef"}, + {"9f11fc131123f844c1226f429b6a0a6af0525d9f40f056c7fc16cdf1b06bda08e302554417a59fa7dcf6247421959d22", "abcdefg"}, + {"9000cd7cada59d1d2eb82912f7f24e5e69cc5517f68283b005fa27c285b61e05edf1ad1a8a9bded6fd29eb87d75ad806", "abcdefgh"}, + {"ef54915b60cf062b8dd0c29ae3cad69abe6310de63ac081f46ef019c5c90897caefd79b796cfa81139788a260ded52df", "abcdefghi"}, + {"a12070030a02d86b0ddacd0d3a5b598344513d0a051e7355053e556a0055489c1555399b03342845c4adde2dc44ff66c", "abcdefghij"}, + {"86f58ec2d74d1b7f8eb0c2ff0967316699639e8d4eb129de54bdf34c96cdbabe200d052149f2dd787f43571ba74670d4", "Discard medicine more than two years old."}, + {"ae4a2b639ca9bfa04b1855d5a05fe7f230994f790891c6979103e2605f660c4c1262a48142dcbeb57a1914ba5f7c3fa7", "He who has a shady past knows that nice guys finish last."}, + {"40ae213df6436eca952aa6841886fcdb82908ef1576a99c8f49bb9dd5023169f7c53035abdda0b54c302f4974e2105e7", "I wouldn't marry him with a ten foot pole."}, + {"e7cf8b873c9bc950f06259aa54309f349cefa72c00d597aebf903e6519a50011dfe355afff064a10701c705693848df9", "Free! Free!/A trip/to Mars/for 900/empty jars/Burma Shave"}, + {"c3d4f0f4047181c7d39d34703365f7bf70207183caf2c2f6145f04da895ef69124d9cdeb635da636c3a474e61024e29b", "The days of the digital watch are numbered. -Tom Stoppard"}, + {"a097aab567e167d5cf93676ed73252a69f9687cb3179bb2d27c9878119e94bf7b7c4b58dc90582edfaf66e11388ed714", "Nepal premier won't resign."}, + {"5026ca45c41fc64712eb65065da92f6467541c78f8966d3fe2c8e3fb769a3ec14215f819654b47bd64f7f0eac17184f3", "For every action there is an equal and opposite government program."}, + {"ac1cc0f5ac8d5f5514a7b738ac322b7fb52a161b449c3672e9b6a6ad1a5e4b26b001cf3bad24c56598676ca17d4b445a", "His money is twice tainted: 'taint yours and 'taint mine."}, + {"722d10c5de371ec0c8c4b5247ac8a5f1d240d68c73f8da13d8b25f0166d6f309bf9561979a111a0049405771d201941a", "There is no reason for any individual to have a computer in their home. -Ken Olsen, 1977"}, + {"dc2d3ea18bfa10549c63bf2b75b39b5167a80c12aff0e05443168ea87ff149fb0eda5e0bd234eb5d48c7d02ffc5807f1", "It's a tiny change to the code and not completely disgusting. - Bob Manchek"}, + {"1d67c969e2a945ae5346d2139760261504d4ba164c522443afe19ef3e29b152a4c52445489cfc9d7215e5a450e8e1e4e", "size: a.out: bad magic"}, + {"5ff8e075e465646e7b73ef36d812c6e9f7d60fa6ea0e533e5569b4f73cde53cdd2cc787f33540af57cca3fe467d32fe0", "The major problem is with sendmail. -Mark Horton"}, + {"5bd0a997a67c9ae1979a894eb0cde403dde003c9b6f2c03cf21925c42ff4e1176e6df1ca005381612ef18457b9b7ec3b", "Give me a rock, paper and scissors and I will move the world. CCFestoon"}, + {"1eee6da33e7e54fc5be52ae23b94b16ba4d2a947ae4505c6a3edfc7401151ea5205ac01b669b56f27d8ef7f175ed7762", "If the enemy is within range, then so are you."}, + {"76b06e9dea66bfbb1a96029426dc0dfd7830bd297eb447ff5358d94a87cd00c88b59df2493fef56ecbb5231073892ea9", "It's well we cannot hear the screams/That we create in others' dreams."}, + {"12acaf21452cff586143e3f5db0bfdf7802c057e1adf2a619031c4e1b0ccc4208cf6cef8fe722bbaa2fb46a30d9135d8", "You remind me of a TV show, but that's all right: I watch it anyway."}, + {"0fc23d7f4183efd186f0bc4fc5db867e026e2146b06cb3d52f4bdbd57d1740122caa853b41868b197b2ac759db39df88", "C is as portable as Stonehedge!!"}, + {"bc805578a7f85d34a86a32976e1c34fe65cf815186fbef76f46ef99cda10723f971f3f1464d488243f5e29db7488598d", "Even if I could be Shakespeare, I think I should still choose to be Faraday. - A. Huxley"}, + {"b23918399a12ebf4431559eec3813eaf7412e875fd7464f16d581e473330842d2e96c6be49a7ce3f9bb0b8bc0fcbe0fe", "The fugacity of a constituent in a mixture of gases at a given temperature is proportional to its mole fraction. Lewis-Randall Rule"}, + {"1764b700eb1ead52a2fc33cc28975c2180f1b8faa5038d94cffa8d78154aab16e91dd787e7b0303948ebed62561542c8", "How can you write a big system without C++? -Paul Glick"}, +} + +func TestGolden(t *testing.T) { + for i := 0; i < len(golden); i++ { + g := golden[i] + c := New() + for j := 0; j < 3; j++ { + if j < 2 { + io.WriteString(c, g.in) + } else { + io.WriteString(c, g.in[0:len(g.in)/2]) + c.Sum() + io.WriteString(c, g.in[len(g.in)/2:]) + } + s := fmt.Sprintf("%x", c.Sum()) + if s != g.out { + t.Fatalf("sha512[%d](%s) = %s want %s", j, g.in, s, g.out) + } + c.Reset() + } + } + for i := 0; i < len(golden384); i++ { + g := golden384[i] + c := New384() + for j := 0; j < 3; j++ { + if j < 2 { + io.WriteString(c, g.in) + } else { + io.WriteString(c, g.in[0:len(g.in)/2]) + c.Sum() + io.WriteString(c, g.in[len(g.in)/2:]) + } + s := fmt.Sprintf("%x", c.Sum()) + if s != g.out { + t.Fatalf("sha384[%d](%s) = %s want %s", j, g.in, s, g.out) + } + c.Reset() + } + } +} diff --git a/libgo/go/crypto/sha512/sha512block.go b/libgo/go/crypto/sha512/sha512block.go new file mode 100644 index 000000000..6b7506287 --- /dev/null +++ b/libgo/go/crypto/sha512/sha512block.go @@ -0,0 +1,144 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// SHA512 block step. +// In its own file so that a faster assembly or C version +// can be substituted easily. + +package sha512 + +var _K = []uint64{ + 0x428a2f98d728ae22, + 0x7137449123ef65cd, + 0xb5c0fbcfec4d3b2f, + 0xe9b5dba58189dbbc, + 0x3956c25bf348b538, + 0x59f111f1b605d019, + 0x923f82a4af194f9b, + 0xab1c5ed5da6d8118, + 0xd807aa98a3030242, + 0x12835b0145706fbe, + 0x243185be4ee4b28c, + 0x550c7dc3d5ffb4e2, + 0x72be5d74f27b896f, + 0x80deb1fe3b1696b1, + 0x9bdc06a725c71235, + 0xc19bf174cf692694, + 0xe49b69c19ef14ad2, + 0xefbe4786384f25e3, + 0x0fc19dc68b8cd5b5, + 0x240ca1cc77ac9c65, + 0x2de92c6f592b0275, + 0x4a7484aa6ea6e483, + 0x5cb0a9dcbd41fbd4, + 0x76f988da831153b5, + 0x983e5152ee66dfab, + 0xa831c66d2db43210, + 0xb00327c898fb213f, + 0xbf597fc7beef0ee4, + 0xc6e00bf33da88fc2, + 0xd5a79147930aa725, + 0x06ca6351e003826f, + 0x142929670a0e6e70, + 0x27b70a8546d22ffc, + 0x2e1b21385c26c926, + 0x4d2c6dfc5ac42aed, + 0x53380d139d95b3df, + 0x650a73548baf63de, + 0x766a0abb3c77b2a8, + 0x81c2c92e47edaee6, + 0x92722c851482353b, + 0xa2bfe8a14cf10364, + 0xa81a664bbc423001, + 0xc24b8b70d0f89791, + 0xc76c51a30654be30, + 0xd192e819d6ef5218, + 0xd69906245565a910, + 0xf40e35855771202a, + 0x106aa07032bbd1b8, + 0x19a4c116b8d2d0c8, + 0x1e376c085141ab53, + 0x2748774cdf8eeb99, + 0x34b0bcb5e19b48a8, + 0x391c0cb3c5c95a63, + 0x4ed8aa4ae3418acb, + 0x5b9cca4f7763e373, + 0x682e6ff3d6b2b8a3, + 0x748f82ee5defb2fc, + 0x78a5636f43172f60, + 0x84c87814a1f0ab72, + 0x8cc702081a6439ec, + 0x90befffa23631e28, + 0xa4506cebde82bde9, + 0xbef9a3f7b2c67915, + 0xc67178f2e372532b, + 0xca273eceea26619c, + 0xd186b8c721c0c207, + 0xeada7dd6cde0eb1e, + 0xf57d4f7fee6ed178, + 0x06f067aa72176fba, + 0x0a637dc5a2c898a6, + 0x113f9804bef90dae, + 0x1b710b35131c471b, + 0x28db77f523047d84, + 0x32caab7b40c72493, + 0x3c9ebe0a15c9bebc, + 0x431d67c49c100d4c, + 0x4cc5d4becb3e42b6, + 0x597f299cfc657e2a, + 0x5fcb6fab3ad6faec, + 0x6c44198c4a475817, +} + +func _Block(dig *digest, p []byte) int { + var w [80]uint64 + n := 0 + h0, h1, h2, h3, h4, h5, h6, h7 := dig.h[0], dig.h[1], dig.h[2], dig.h[3], dig.h[4], dig.h[5], dig.h[6], dig.h[7] + for len(p) >= _Chunk { + for i := 0; i < 16; i++ { + j := i * 8 + w[i] = uint64(p[j])<<56 | uint64(p[j+1])<<48 | uint64(p[j+2])<<40 | uint64(p[j+3])<<32 | + uint64(p[j+4])<<24 | uint64(p[j+5])<<16 | uint64(p[j+6])<<8 | uint64(p[j+7]) + } + for i := 16; i < 80; i++ { + t1 := (w[i-2]>>19 | w[i-2]<<(64-19)) ^ (w[i-2]>>61 | w[i-2]<<(64-61)) ^ (w[i-2] >> 6) + + t2 := (w[i-15]>>1 | w[i-15]<<(64-1)) ^ (w[i-15]>>8 | w[i-15]<<(64-8)) ^ (w[i-15] >> 7) + + w[i] = t1 + w[i-7] + t2 + w[i-16] + } + + a, b, c, d, e, f, g, h := h0, h1, h2, h3, h4, h5, h6, h7 + + for i := 0; i < 80; i++ { + t1 := h + ((e>>14 | e<<(64-14)) ^ (e>>18 | e<<(64-18)) ^ (e>>41 | e<<(64-41))) + ((e & f) ^ (^e & g)) + _K[i] + w[i] + + t2 := ((a>>28 | a<<(64-28)) ^ (a>>34 | a<<(64-34)) ^ (a>>39 | a<<(64-39))) + ((a & b) ^ (a & c) ^ (b & c)) + + h = g + g = f + f = e + e = d + t1 + d = c + c = b + b = a + a = t1 + t2 + } + + h0 += a + h1 += b + h2 += c + h3 += d + h4 += e + h5 += f + h6 += g + h7 += h + + p = p[_Chunk:] + n += _Chunk + } + + dig.h[0], dig.h[1], dig.h[2], dig.h[3], dig.h[4], dig.h[5], dig.h[6], dig.h[7] = h0, h1, h2, h3, h4, h5, h6, h7 + return n +} diff --git a/libgo/go/crypto/subtle/constant_time.go b/libgo/go/crypto/subtle/constant_time.go new file mode 100644 index 000000000..a3d70b9c9 --- /dev/null +++ b/libgo/go/crypto/subtle/constant_time.go @@ -0,0 +1,57 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This package implements functions that are often useful in cryptographic +// code but require careful thought to use correctly. +package subtle + +// ConstantTimeCompare returns 1 iff the two equal length slices, x +// and y, have equal contents. The time taken is a function of the length of +// the slices and is independent of the contents. +func ConstantTimeCompare(x, y []byte) int { + var v byte + + for i := 0; i < len(x); i++ { + v |= x[i] ^ y[i] + } + + return ConstantTimeByteEq(v, 0) +} + +// ConstantTimeSelect returns x if v is 1 and y if v is 0. +// Its behavior is undefined if v takes any other value. +func ConstantTimeSelect(v, x, y int) int { return ^(v-1)&x | (v-1)&y } + +// ConstantTimeByteEq returns 1 if x == y and 0 otherwise. +func ConstantTimeByteEq(x, y uint8) int { + z := ^(x ^ y) + z &= z >> 4 + z &= z >> 2 + z &= z >> 1 + + return int(z) +} + +// ConstantTimeEq returns 1 if x == y and 0 otherwise. +func ConstantTimeEq(x, y int32) int { + z := ^(x ^ y) + z &= z >> 16 + z &= z >> 8 + z &= z >> 4 + z &= z >> 2 + z &= z >> 1 + + return int(z & 1) +} + +// ConstantTimeCopy copies the contents of y into x iff v == 1. If v == 0, x is left unchanged. +// Its behavior is undefined if v takes any other value. +func ConstantTimeCopy(v int, x, y []byte) { + xmask := byte(v - 1) + ymask := byte(^(v - 1)) + for i := 0; i < len(x); i++ { + x[i] = x[i]&xmask | y[i]&ymask + } + return +} diff --git a/libgo/go/crypto/subtle/constant_time_test.go b/libgo/go/crypto/subtle/constant_time_test.go new file mode 100644 index 000000000..b28b73581 --- /dev/null +++ b/libgo/go/crypto/subtle/constant_time_test.go @@ -0,0 +1,105 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package subtle + +import ( + "testing" + "testing/quick" +) + +type TestConstantTimeCompareStruct struct { + a, b []byte + out int +} + +var testConstandTimeCompareData = []TestConstantTimeCompareStruct{ + {[]byte{}, []byte{}, 1}, + {[]byte{0x11}, []byte{0x11}, 1}, + {[]byte{0x12}, []byte{0x11}, 0}, +} + +func TestConstantTimeCompare(t *testing.T) { + for i, test := range testConstandTimeCompareData { + if r := ConstantTimeCompare(test.a, test.b); r != test.out { + t.Errorf("#%d bad result (got %x, want %x)", i, r, test.out) + } + } +} + +type TestConstantTimeByteEqStruct struct { + a, b uint8 + out int +} + +var testConstandTimeByteEqData = []TestConstantTimeByteEqStruct{ + {0, 0, 1}, + {0, 1, 0}, + {1, 0, 0}, + {0xff, 0xff, 1}, + {0xff, 0xfe, 0}, +} + +func byteEq(a, b uint8) int { + if a == b { + return 1 + } + return 0 +} + +func TestConstantTimeByteEq(t *testing.T) { + for i, test := range testConstandTimeByteEqData { + if r := ConstantTimeByteEq(test.a, test.b); r != test.out { + t.Errorf("#%d bad result (got %x, want %x)", i, r, test.out) + } + } + err := quick.CheckEqual(ConstantTimeByteEq, byteEq, nil) + if err != nil { + t.Error(err) + } +} + +func eq(a, b int32) int { + if a == b { + return 1 + } + return 0 +} + +func TestConstantTimeEq(t *testing.T) { + err := quick.CheckEqual(ConstantTimeEq, eq, nil) + if err != nil { + t.Error(err) + } +} + +func makeCopy(v int, x, y []byte) []byte { + if len(x) > len(y) { + x = x[0:len(y)] + } else { + y = y[0:len(x)] + } + if v == 1 { + copy(x, y) + } + return x +} + +func constantTimeCopyWrapper(v int, x, y []byte) []byte { + if len(x) > len(y) { + x = x[0:len(y)] + } else { + y = y[0:len(x)] + } + v &= 1 + ConstantTimeCopy(v, x, y) + return x +} + +func TestConstantTimeCopy(t *testing.T) { + err := quick.CheckEqual(constantTimeCopyWrapper, makeCopy, nil) + if err != nil { + t.Error(err) + } +} diff --git a/libgo/go/crypto/tls/alert.go b/libgo/go/crypto/tls/alert.go new file mode 100644 index 000000000..3b9e0e241 --- /dev/null +++ b/libgo/go/crypto/tls/alert.go @@ -0,0 +1,73 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tls + +import "strconv" + +type alert uint8 + +const ( + // alert level + alertLevelWarning = 1 + alertLevelError = 2 +) + +const ( + alertCloseNotify alert = 0 + alertUnexpectedMessage alert = 10 + alertBadRecordMAC alert = 20 + alertDecryptionFailed alert = 21 + alertRecordOverflow alert = 22 + alertDecompressionFailure alert = 30 + alertHandshakeFailure alert = 40 + alertBadCertificate alert = 42 + alertUnsupportedCertificate alert = 43 + alertCertificateRevoked alert = 44 + alertCertificateExpired alert = 45 + alertCertificateUnknown alert = 46 + alertIllegalParameter alert = 47 + alertUnknownCA alert = 48 + alertAccessDenied alert = 49 + alertDecodeError alert = 50 + alertDecryptError alert = 51 + alertProtocolVersion alert = 70 + alertInsufficientSecurity alert = 71 + alertInternalError alert = 80 + alertUserCanceled alert = 90 + alertNoRenegotiation alert = 100 +) + +var alertText = map[alert]string{ + alertCloseNotify: "close notify", + alertUnexpectedMessage: "unexpected message", + alertBadRecordMAC: "bad record MAC", + alertDecryptionFailed: "decryption failed", + alertRecordOverflow: "record overflow", + alertDecompressionFailure: "decompression failure", + alertHandshakeFailure: "handshake failure", + alertBadCertificate: "bad certificate", + alertUnsupportedCertificate: "unsupported certificate", + alertCertificateRevoked: "revoked certificate", + alertCertificateExpired: "expired certificate", + alertCertificateUnknown: "unknown certificate", + alertIllegalParameter: "illegal parameter", + alertUnknownCA: "unknown certificate authority", + alertAccessDenied: "access denied", + alertDecodeError: "error decoding message", + alertDecryptError: "error decrypting message", + alertProtocolVersion: "protocol version not supported", + alertInsufficientSecurity: "insufficient security level", + alertInternalError: "internal error", + alertUserCanceled: "user canceled", + alertNoRenegotiation: "no renegotiation", +} + +func (e alert) String() string { + s, ok := alertText[e] + if ok { + return s + } + return "alert(" + strconv.Itoa(int(e)) + ")" +} diff --git a/libgo/go/crypto/tls/ca_set.go b/libgo/go/crypto/tls/ca_set.go new file mode 100644 index 000000000..ae00ac558 --- /dev/null +++ b/libgo/go/crypto/tls/ca_set.go @@ -0,0 +1,89 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tls + +import ( + "crypto/x509" + "encoding/pem" + "strings" +) + +// A CASet is a set of certificates. +type CASet struct { + bySubjectKeyId map[string][]*x509.Certificate + byName map[string][]*x509.Certificate +} + +// NewCASet returns a new, empty CASet. +func NewCASet() *CASet { + return &CASet{ + make(map[string][]*x509.Certificate), + make(map[string][]*x509.Certificate), + } +} + +func nameToKey(name *x509.Name) string { + return strings.Join(name.Country, ",") + "/" + strings.Join(name.Organization, ",") + "/" + strings.Join(name.OrganizationalUnit, ",") + "/" + name.CommonName +} + +// FindVerifiedParent attempts to find the certificate in s which has signed +// the given certificate. If no such certificate can be found or the signature +// doesn't match, it returns nil. +func (s *CASet) FindVerifiedParent(cert *x509.Certificate) (parent *x509.Certificate) { + var candidates []*x509.Certificate + + if len(cert.AuthorityKeyId) > 0 { + candidates = s.bySubjectKeyId[string(cert.AuthorityKeyId)] + } + if len(candidates) == 0 { + candidates = s.byName[nameToKey(&cert.Issuer)] + } + + for _, c := range candidates { + if cert.CheckSignatureFrom(c) == nil { + return c + } + } + + return nil +} + +// AddCert adds a certificate to the set +func (s *CASet) AddCert(cert *x509.Certificate) { + if len(cert.SubjectKeyId) > 0 { + keyId := string(cert.SubjectKeyId) + s.bySubjectKeyId[keyId] = append(s.bySubjectKeyId[keyId], cert) + } + name := nameToKey(&cert.Subject) + s.byName[name] = append(s.byName[name], cert) +} + +// SetFromPEM attempts to parse a series of PEM encoded root certificates. It +// appends any certificates found to s and returns true if any certificates +// were successfully parsed. On many Linux systems, /etc/ssl/cert.pem will +// contains the system wide set of root CAs in a format suitable for this +// function. +func (s *CASet) SetFromPEM(pemCerts []byte) (ok bool) { + for len(pemCerts) > 0 { + var block *pem.Block + block, pemCerts = pem.Decode(pemCerts) + if block == nil { + break + } + if block.Type != "CERTIFICATE" || len(block.Headers) != 0 { + continue + } + + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + continue + } + + s.AddCert(cert) + ok = true + } + + return +} diff --git a/libgo/go/crypto/tls/cipher_suites.go b/libgo/go/crypto/tls/cipher_suites.go new file mode 100644 index 000000000..bc7b0d32f --- /dev/null +++ b/libgo/go/crypto/tls/cipher_suites.go @@ -0,0 +1,102 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tls + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/hmac" + "crypto/rc4" + "crypto/x509" + "hash" + "os" +) + +// a keyAgreement implements the client and server side of a TLS key agreement +// protocol by generating and processing key exchange messages. +type keyAgreement interface { + // On the server side, the first two methods are called in order. + + // In the case that the key agreement protocol doesn't use a + // ServerKeyExchange message, generateServerKeyExchange can return nil, + // nil. + generateServerKeyExchange(*Config, *clientHelloMsg, *serverHelloMsg) (*serverKeyExchangeMsg, os.Error) + processClientKeyExchange(*Config, *clientKeyExchangeMsg) ([]byte, os.Error) + + // On the client side, the next two methods are called in order. + + // This method may not be called if the server doesn't send a + // ServerKeyExchange message. + processServerKeyExchange(*Config, *clientHelloMsg, *serverHelloMsg, *x509.Certificate, *serverKeyExchangeMsg) os.Error + generateClientKeyExchange(*Config, *clientHelloMsg, *x509.Certificate) ([]byte, *clientKeyExchangeMsg, os.Error) +} + +// A cipherSuite is a specific combination of key agreement, cipher and MAC +// function. All cipher suites currently assume RSA key agreement. +type cipherSuite struct { + // the lengths, in bytes, of the key material needed for each component. + keyLen int + macLen int + ivLen int + ka func() keyAgreement + // If elliptic is set, a server will only consider this ciphersuite if + // the ClientHello indicated that the client supports an elliptic curve + // and point format that we can handle. + elliptic bool + cipher func(key, iv []byte, isRead bool) interface{} + mac func(macKey []byte) hash.Hash +} + +var cipherSuites = map[uint16]*cipherSuite{ + TLS_RSA_WITH_RC4_128_SHA: &cipherSuite{16, 20, 0, rsaKA, false, cipherRC4, hmacSHA1}, + TLS_RSA_WITH_AES_128_CBC_SHA: &cipherSuite{16, 20, 16, rsaKA, false, cipherAES, hmacSHA1}, + TLS_ECDHE_RSA_WITH_RC4_128_SHA: &cipherSuite{16, 20, 0, ecdheRSAKA, true, cipherRC4, hmacSHA1}, + TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA: &cipherSuite{16, 20, 16, ecdheRSAKA, true, cipherAES, hmacSHA1}, +} + +func cipherRC4(key, iv []byte, isRead bool) interface{} { + cipher, _ := rc4.NewCipher(key) + return cipher +} + +func cipherAES(key, iv []byte, isRead bool) interface{} { + block, _ := aes.NewCipher(key) + if isRead { + return cipher.NewCBCDecrypter(block, iv) + } + return cipher.NewCBCEncrypter(block, iv) +} + +func hmacSHA1(key []byte) hash.Hash { + return hmac.NewSHA1(key) +} + +func rsaKA() keyAgreement { + return rsaKeyAgreement{} +} + +func ecdheRSAKA() keyAgreement { + return new(ecdheRSAKeyAgreement) +} + +// mutualCipherSuite returns a cipherSuite and its id given a list of supported +// ciphersuites and the id requested by the peer. +func mutualCipherSuite(have []uint16, want uint16) (suite *cipherSuite, id uint16) { + for _, id := range have { + if id == want { + return cipherSuites[id], id + } + } + return +} + +// A list of the possible cipher suite ids. Taken from +// http://www.iana.org/assignments/tls-parameters/tls-parameters.xml +const ( + TLS_RSA_WITH_RC4_128_SHA uint16 = 0x0005 + TLS_RSA_WITH_AES_128_CBC_SHA uint16 = 0x002f + TLS_ECDHE_RSA_WITH_RC4_128_SHA uint16 = 0xc011 + TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA uint16 = 0xc013 +) diff --git a/libgo/go/crypto/tls/common.go b/libgo/go/crypto/tls/common.go new file mode 100644 index 000000000..7135f3d0f --- /dev/null +++ b/libgo/go/crypto/tls/common.go @@ -0,0 +1,258 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tls + +import ( + "crypto/rand" + "crypto/rsa" + "io" + "io/ioutil" + "sync" + "time" +) + +const ( + maxPlaintext = 16384 // maximum plaintext payload length + maxCiphertext = 16384 + 2048 // maximum ciphertext payload length + recordHeaderLen = 5 // record header length + maxHandshake = 65536 // maximum handshake we support (protocol max is 16 MB) + + minVersion = 0x0301 // minimum supported version - TLS 1.0 + maxVersion = 0x0301 // maximum supported version - TLS 1.0 +) + +// TLS record types. +type recordType uint8 + +const ( + recordTypeChangeCipherSpec recordType = 20 + recordTypeAlert recordType = 21 + recordTypeHandshake recordType = 22 + recordTypeApplicationData recordType = 23 +) + +// TLS handshake message types. +const ( + typeClientHello uint8 = 1 + typeServerHello uint8 = 2 + typeCertificate uint8 = 11 + typeServerKeyExchange uint8 = 12 + typeCertificateRequest uint8 = 13 + typeServerHelloDone uint8 = 14 + typeCertificateVerify uint8 = 15 + typeClientKeyExchange uint8 = 16 + typeFinished uint8 = 20 + typeCertificateStatus uint8 = 22 + typeNextProtocol uint8 = 67 // Not IANA assigned +) + +// TLS compression types. +const ( + compressionNone uint8 = 0 +) + +// TLS extension numbers +var ( + extensionServerName uint16 = 0 + extensionStatusRequest uint16 = 5 + extensionSupportedCurves uint16 = 10 + extensionSupportedPoints uint16 = 11 + extensionNextProtoNeg uint16 = 13172 // not IANA assigned +) + +// TLS Elliptic Curves +// http://www.iana.org/assignments/tls-parameters/tls-parameters.xml#tls-parameters-8 +var ( + curveP256 uint16 = 23 + curveP384 uint16 = 24 + curveP521 uint16 = 25 +) + +// TLS Elliptic Curve Point Formats +// http://www.iana.org/assignments/tls-parameters/tls-parameters.xml#tls-parameters-9 +var ( + pointFormatUncompressed uint8 = 0 +) + +// TLS CertificateStatusType (RFC 3546) +const ( + statusTypeOCSP uint8 = 1 +) + +// Certificate types (for certificateRequestMsg) +const ( + certTypeRSASign = 1 // A certificate containing an RSA key + certTypeDSSSign = 2 // A certificate containing a DSA key + certTypeRSAFixedDH = 3 // A certificate containing a static DH key + certTypeDSSFixedDH = 4 // A certficiate containing a static DH key + // Rest of these are reserved by the TLS spec +) + +// ConnectionState records basic TLS details about the connection. +type ConnectionState struct { + HandshakeComplete bool + CipherSuite uint16 + NegotiatedProtocol string +} + +// A Config structure is used to configure a TLS client or server. After one +// has been passed to a TLS function it must not be modified. +type Config struct { + // Rand provides the source of entropy for nonces and RSA blinding. + // If Rand is nil, TLS uses the cryptographic random reader in package + // crypto/rand. + Rand io.Reader + + // Time returns the current time as the number of seconds since the epoch. + // If Time is nil, TLS uses the system time.Seconds. + Time func() int64 + + // Certificates contains one or more certificate chains + // to present to the other side of the connection. + // Server configurations must include at least one certificate. + Certificates []Certificate + + // RootCAs defines the set of root certificate authorities + // that clients use when verifying server certificates. + // If RootCAs is nil, TLS uses the host's root CA set. + RootCAs *CASet + + // NextProtos is a list of supported, application level protocols. + // Currently only server-side handling is supported. + NextProtos []string + + // ServerName is included in the client's handshake to support virtual + // hosting. + ServerName string + + // AuthenticateClient controls whether a server will request a certificate + // from the client. It does not require that the client send a + // certificate nor does it require that the certificate sent be + // anything more than self-signed. + AuthenticateClient bool + + // CipherSuites is a list of supported cipher suites. If CipherSuites + // is nil, TLS uses a list of suites supported by the implementation. + CipherSuites []uint16 +} + +func (c *Config) rand() io.Reader { + r := c.Rand + if r == nil { + return rand.Reader + } + return r +} + +func (c *Config) time() int64 { + t := c.Time + if t == nil { + t = time.Seconds + } + return t() +} + +func (c *Config) rootCAs() *CASet { + s := c.RootCAs + if s == nil { + s = defaultRoots() + } + return s +} + +func (c *Config) cipherSuites() []uint16 { + s := c.CipherSuites + if s == nil { + s = defaultCipherSuites() + } + return s +} + +// A Certificate is a chain of one or more certificates, leaf first. +type Certificate struct { + Certificate [][]byte + PrivateKey *rsa.PrivateKey +} + +// A TLS record. +type record struct { + contentType recordType + major, minor uint8 + payload []byte +} + +type handshakeMessage interface { + marshal() []byte + unmarshal([]byte) bool +} + +// mutualVersion returns the protocol version to use given the advertised +// version of the peer. +func mutualVersion(vers uint16) (uint16, bool) { + if vers < minVersion { + return 0, false + } + if vers > maxVersion { + vers = maxVersion + } + return vers, true +} + +var emptyConfig Config + +func defaultConfig() *Config { + return &emptyConfig +} + +// Possible certificate files; stop after finding one. +// On OS X we should really be using the Directory Services keychain +// but that requires a lot of Mach goo to get at. Instead we use +// the same root set that curl uses. +var certFiles = []string{ + "/etc/ssl/certs/ca-certificates.crt", // Linux etc + "/usr/share/curl/curl-ca-bundle.crt", // OS X +} + +var once sync.Once + +func defaultRoots() *CASet { + once.Do(initDefaults) + return varDefaultRoots +} + +func defaultCipherSuites() []uint16 { + once.Do(initDefaults) + return varDefaultCipherSuites +} + +func initDefaults() { + initDefaultRoots() + initDefaultCipherSuites() +} + +var varDefaultRoots *CASet + +func initDefaultRoots() { + roots := NewCASet() + for _, file := range certFiles { + data, err := ioutil.ReadFile(file) + if err == nil { + roots.SetFromPEM(data) + break + } + } + varDefaultRoots = roots +} + +var varDefaultCipherSuites []uint16 + +func initDefaultCipherSuites() { + varDefaultCipherSuites = make([]uint16, len(cipherSuites)) + i := 0 + for id, _ := range cipherSuites { + varDefaultCipherSuites[i] = id + i++ + } +} diff --git a/libgo/go/crypto/tls/conn.go b/libgo/go/crypto/tls/conn.go new file mode 100644 index 000000000..d203e8d51 --- /dev/null +++ b/libgo/go/crypto/tls/conn.go @@ -0,0 +1,801 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// TLS low level connection and record layer + +package tls + +import ( + "bytes" + "crypto/cipher" + "crypto/subtle" + "crypto/x509" + "hash" + "io" + "net" + "os" + "sync" +) + +// A Conn represents a secured connection. +// It implements the net.Conn interface. +type Conn struct { + // constant + conn net.Conn + isClient bool + + // constant after handshake; protected by handshakeMutex + handshakeMutex sync.Mutex // handshakeMutex < in.Mutex, out.Mutex, errMutex + vers uint16 // TLS version + haveVers bool // version has been negotiated + config *Config // configuration passed to constructor + handshakeComplete bool + cipherSuite uint16 + ocspResponse []byte // stapled OCSP response + peerCertificates []*x509.Certificate + + clientProtocol string + + // first permanent error + errMutex sync.Mutex + err os.Error + + // input/output + in, out halfConn // in.Mutex < out.Mutex + rawInput *block // raw input, right off the wire + input *block // application data waiting to be read + hand bytes.Buffer // handshake data waiting to be read + + tmp [16]byte +} + +func (c *Conn) setError(err os.Error) os.Error { + c.errMutex.Lock() + defer c.errMutex.Unlock() + + if c.err == nil { + c.err = err + } + return err +} + +func (c *Conn) error() os.Error { + c.errMutex.Lock() + defer c.errMutex.Unlock() + + return c.err +} + +// Access to net.Conn methods. +// Cannot just embed net.Conn because that would +// export the struct field too. + +// LocalAddr returns the local network address. +func (c *Conn) LocalAddr() net.Addr { + return c.conn.LocalAddr() +} + +// RemoteAddr returns the remote network address. +func (c *Conn) RemoteAddr() net.Addr { + return c.conn.RemoteAddr() +} + +// SetTimeout sets the read deadline associated with the connection. +// There is no write deadline. +func (c *Conn) SetTimeout(nsec int64) os.Error { + return c.conn.SetTimeout(nsec) +} + +// SetReadTimeout sets the time (in nanoseconds) that +// Read will wait for data before returning os.EAGAIN. +// Setting nsec == 0 (the default) disables the deadline. +func (c *Conn) SetReadTimeout(nsec int64) os.Error { + return c.conn.SetReadTimeout(nsec) +} + +// SetWriteTimeout exists to satisfy the net.Conn interface +// but is not implemented by TLS. It always returns an error. +func (c *Conn) SetWriteTimeout(nsec int64) os.Error { + return os.NewError("TLS does not support SetWriteTimeout") +} + +// A halfConn represents one direction of the record layer +// connection, either sending or receiving. +type halfConn struct { + sync.Mutex + cipher interface{} // cipher algorithm + mac hash.Hash // MAC algorithm + seq [8]byte // 64-bit sequence number + bfree *block // list of free blocks + + nextCipher interface{} // next encryption state + nextMac hash.Hash // next MAC algorithm +} + +// prepareCipherSpec sets the encryption and MAC states +// that a subsequent changeCipherSpec will use. +func (hc *halfConn) prepareCipherSpec(cipher interface{}, mac hash.Hash) { + hc.nextCipher = cipher + hc.nextMac = mac +} + +// changeCipherSpec changes the encryption and MAC states +// to the ones previously passed to prepareCipherSpec. +func (hc *halfConn) changeCipherSpec() os.Error { + if hc.nextCipher == nil { + return alertInternalError + } + hc.cipher = hc.nextCipher + hc.mac = hc.nextMac + hc.nextCipher = nil + hc.nextMac = nil + return nil +} + +// incSeq increments the sequence number. +func (hc *halfConn) incSeq() { + for i := 7; i >= 0; i-- { + hc.seq[i]++ + if hc.seq[i] != 0 { + return + } + } + + // Not allowed to let sequence number wrap. + // Instead, must renegotiate before it does. + // Not likely enough to bother. + panic("TLS: sequence number wraparound") +} + +// resetSeq resets the sequence number to zero. +func (hc *halfConn) resetSeq() { + for i := range hc.seq { + hc.seq[i] = 0 + } +} + +// removePadding returns an unpadded slice, in constant time, which is a prefix +// of the input. It also returns a byte which is equal to 255 if the padding +// was valid and 0 otherwise. See RFC 2246, section 6.2.3.2 +func removePadding(payload []byte) ([]byte, byte) { + if len(payload) < 1 { + return payload, 0 + } + + paddingLen := payload[len(payload)-1] + t := uint(len(payload)-1) - uint(paddingLen) + // if len(payload) >= (paddingLen - 1) then the MSB of t is zero + good := byte(int32(^t) >> 31) + + toCheck := 255 // the maximum possible padding length + // The length of the padded data is public, so we can use an if here + if toCheck+1 > len(payload) { + toCheck = len(payload) - 1 + } + + for i := 0; i < toCheck; i++ { + t := uint(paddingLen) - uint(i) + // if i <= paddingLen then the MSB of t is zero + mask := byte(int32(^t) >> 31) + b := payload[len(payload)-1-i] + good &^= mask&paddingLen ^ mask&b + } + + // We AND together the bits of good and replicate the result across + // all the bits. + good &= good << 4 + good &= good << 2 + good &= good << 1 + good = uint8(int8(good) >> 7) + + toRemove := good&paddingLen + 1 + return payload[:len(payload)-int(toRemove)], good +} + +func roundUp(a, b int) int { + return a + (b-a%b)%b +} + +// decrypt checks and strips the mac and decrypts the data in b. +func (hc *halfConn) decrypt(b *block) (bool, alert) { + // pull out payload + payload := b.data[recordHeaderLen:] + + macSize := 0 + if hc.mac != nil { + macSize = hc.mac.Size() + } + + paddingGood := byte(255) + + // decrypt + if hc.cipher != nil { + switch c := hc.cipher.(type) { + case cipher.Stream: + c.XORKeyStream(payload, payload) + case cipher.BlockMode: + blockSize := c.BlockSize() + + if len(payload)%blockSize != 0 || len(payload) < roundUp(macSize+1, blockSize) { + return false, alertBadRecordMAC + } + + c.CryptBlocks(payload, payload) + payload, paddingGood = removePadding(payload) + b.resize(recordHeaderLen + len(payload)) + + // note that we still have a timing side-channel in the + // MAC check, below. An attacker can align the record + // so that a correct padding will cause one less hash + // block to be calculated. Then they can iteratively + // decrypt a record by breaking each byte. See + // "Password Interception in a SSL/TLS Channel", Brice + // Canvel et al. + // + // However, our behaviour matches OpenSSL, so we leak + // only as much as they do. + default: + panic("unknown cipher type") + } + } + + // check, strip mac + if hc.mac != nil { + if len(payload) < macSize { + return false, alertBadRecordMAC + } + + // strip mac off payload, b.data + n := len(payload) - macSize + b.data[3] = byte(n >> 8) + b.data[4] = byte(n) + b.resize(recordHeaderLen + n) + remoteMAC := payload[n:] + + hc.mac.Reset() + hc.mac.Write(hc.seq[0:]) + hc.incSeq() + hc.mac.Write(b.data) + + if subtle.ConstantTimeCompare(hc.mac.Sum(), remoteMAC) != 1 || paddingGood != 255 { + return false, alertBadRecordMAC + } + } + + return true, 0 +} + +// padToBlockSize calculates the needed padding block, if any, for a payload. +// On exit, prefix aliases payload and extends to the end of the last full +// block of payload. finalBlock is a fresh slice which contains the contents of +// any suffix of payload as well as the needed padding to make finalBlock a +// full block. +func padToBlockSize(payload []byte, blockSize int) (prefix, finalBlock []byte) { + overrun := len(payload) % blockSize + paddingLen := blockSize - overrun + prefix = payload[:len(payload)-overrun] + finalBlock = make([]byte, blockSize) + copy(finalBlock, payload[len(payload)-overrun:]) + for i := overrun; i < blockSize; i++ { + finalBlock[i] = byte(paddingLen - 1) + } + return +} + +// encrypt encrypts and macs the data in b. +func (hc *halfConn) encrypt(b *block) (bool, alert) { + // mac + if hc.mac != nil { + hc.mac.Reset() + hc.mac.Write(hc.seq[0:]) + hc.incSeq() + hc.mac.Write(b.data) + mac := hc.mac.Sum() + n := len(b.data) + b.resize(n + len(mac)) + copy(b.data[n:], mac) + } + + payload := b.data[recordHeaderLen:] + + // encrypt + if hc.cipher != nil { + switch c := hc.cipher.(type) { + case cipher.Stream: + c.XORKeyStream(payload, payload) + case cipher.BlockMode: + prefix, finalBlock := padToBlockSize(payload, c.BlockSize()) + b.resize(recordHeaderLen + len(prefix) + len(finalBlock)) + c.CryptBlocks(b.data[recordHeaderLen:], prefix) + c.CryptBlocks(b.data[recordHeaderLen+len(prefix):], finalBlock) + default: + panic("unknown cipher type") + } + } + + // update length to include MAC and any block padding needed. + n := len(b.data) - recordHeaderLen + b.data[3] = byte(n >> 8) + b.data[4] = byte(n) + + return true, 0 +} + +// A block is a simple data buffer. +type block struct { + data []byte + off int // index for Read + link *block +} + +// resize resizes block to be n bytes, growing if necessary. +func (b *block) resize(n int) { + if n > cap(b.data) { + b.reserve(n) + } + b.data = b.data[0:n] +} + +// reserve makes sure that block contains a capacity of at least n bytes. +func (b *block) reserve(n int) { + if cap(b.data) >= n { + return + } + m := cap(b.data) + if m == 0 { + m = 1024 + } + for m < n { + m *= 2 + } + data := make([]byte, len(b.data), m) + copy(data, b.data) + b.data = data +} + +// readFromUntil reads from r into b until b contains at least n bytes +// or else returns an error. +func (b *block) readFromUntil(r io.Reader, n int) os.Error { + // quick case + if len(b.data) >= n { + return nil + } + + // read until have enough. + b.reserve(n) + for { + m, err := r.Read(b.data[len(b.data):cap(b.data)]) + b.data = b.data[0 : len(b.data)+m] + if len(b.data) >= n { + break + } + if err != nil { + return err + } + } + return nil +} + +func (b *block) Read(p []byte) (n int, err os.Error) { + n = copy(p, b.data[b.off:]) + b.off += n + return +} + +// newBlock allocates a new block, from hc's free list if possible. +func (hc *halfConn) newBlock() *block { + b := hc.bfree + if b == nil { + return new(block) + } + hc.bfree = b.link + b.link = nil + b.resize(0) + return b +} + +// freeBlock returns a block to hc's free list. +// The protocol is such that each side only has a block or two on +// its free list at a time, so there's no need to worry about +// trimming the list, etc. +func (hc *halfConn) freeBlock(b *block) { + b.link = hc.bfree + hc.bfree = b +} + +// splitBlock splits a block after the first n bytes, +// returning a block with those n bytes and a +// block with the remaindec. the latter may be nil. +func (hc *halfConn) splitBlock(b *block, n int) (*block, *block) { + if len(b.data) <= n { + return b, nil + } + bb := hc.newBlock() + bb.resize(len(b.data) - n) + copy(bb.data, b.data[n:]) + b.data = b.data[0:n] + return b, bb +} + +// readRecord reads the next TLS record from the connection +// and updates the record layer state. +// c.in.Mutex <= L; c.input == nil. +func (c *Conn) readRecord(want recordType) os.Error { + // Caller must be in sync with connection: + // handshake data if handshake not yet completed, + // else application data. (We don't support renegotiation.) + switch want { + default: + return c.sendAlert(alertInternalError) + case recordTypeHandshake, recordTypeChangeCipherSpec: + if c.handshakeComplete { + return c.sendAlert(alertInternalError) + } + case recordTypeApplicationData: + if !c.handshakeComplete { + return c.sendAlert(alertInternalError) + } + } + +Again: + if c.rawInput == nil { + c.rawInput = c.in.newBlock() + } + b := c.rawInput + + // Read header, payload. + if err := b.readFromUntil(c.conn, recordHeaderLen); err != nil { + // RFC suggests that EOF without an alertCloseNotify is + // an error, but popular web sites seem to do this, + // so we can't make it an error. + // if err == os.EOF { + // err = io.ErrUnexpectedEOF + // } + if e, ok := err.(net.Error); !ok || !e.Temporary() { + c.setError(err) + } + return err + } + typ := recordType(b.data[0]) + vers := uint16(b.data[1])<<8 | uint16(b.data[2]) + n := int(b.data[3])<<8 | int(b.data[4]) + if c.haveVers && vers != c.vers { + return c.sendAlert(alertProtocolVersion) + } + if n > maxCiphertext { + return c.sendAlert(alertRecordOverflow) + } + if err := b.readFromUntil(c.conn, recordHeaderLen+n); err != nil { + if err == os.EOF { + err = io.ErrUnexpectedEOF + } + if e, ok := err.(net.Error); !ok || !e.Temporary() { + c.setError(err) + } + return err + } + + // Process message. + b, c.rawInput = c.in.splitBlock(b, recordHeaderLen+n) + b.off = recordHeaderLen + if ok, err := c.in.decrypt(b); !ok { + return c.sendAlert(err) + } + data := b.data[b.off:] + if len(data) > maxPlaintext { + c.sendAlert(alertRecordOverflow) + c.in.freeBlock(b) + return c.error() + } + + switch typ { + default: + c.sendAlert(alertUnexpectedMessage) + + case recordTypeAlert: + if len(data) != 2 { + c.sendAlert(alertUnexpectedMessage) + break + } + if alert(data[1]) == alertCloseNotify { + c.setError(os.EOF) + break + } + switch data[0] { + case alertLevelWarning: + // drop on the floor + c.in.freeBlock(b) + goto Again + case alertLevelError: + c.setError(&net.OpError{Op: "remote error", Error: alert(data[1])}) + default: + c.sendAlert(alertUnexpectedMessage) + } + + case recordTypeChangeCipherSpec: + if typ != want || len(data) != 1 || data[0] != 1 { + c.sendAlert(alertUnexpectedMessage) + break + } + err := c.in.changeCipherSpec() + if err != nil { + c.sendAlert(err.(alert)) + } + + case recordTypeApplicationData: + if typ != want { + c.sendAlert(alertUnexpectedMessage) + break + } + c.input = b + b = nil + + case recordTypeHandshake: + // TODO(rsc): Should at least pick off connection close. + if typ != want { + return c.sendAlert(alertNoRenegotiation) + } + c.hand.Write(data) + } + + if b != nil { + c.in.freeBlock(b) + } + return c.error() +} + +// sendAlert sends a TLS alert message. +// c.out.Mutex <= L. +func (c *Conn) sendAlertLocked(err alert) os.Error { + c.tmp[0] = alertLevelError + if err == alertNoRenegotiation { + c.tmp[0] = alertLevelWarning + } + c.tmp[1] = byte(err) + c.writeRecord(recordTypeAlert, c.tmp[0:2]) + // closeNotify is a special case in that it isn't an error: + if err != alertCloseNotify { + return c.setError(&net.OpError{Op: "local error", Error: err}) + } + return nil +} + +// sendAlert sends a TLS alert message. +// L < c.out.Mutex. +func (c *Conn) sendAlert(err alert) os.Error { + c.out.Lock() + defer c.out.Unlock() + return c.sendAlertLocked(err) +} + +// writeRecord writes a TLS record with the given type and payload +// to the connection and updates the record layer state. +// c.out.Mutex <= L. +func (c *Conn) writeRecord(typ recordType, data []byte) (n int, err os.Error) { + b := c.out.newBlock() + for len(data) > 0 { + m := len(data) + if m > maxPlaintext { + m = maxPlaintext + } + b.resize(recordHeaderLen + m) + b.data[0] = byte(typ) + vers := c.vers + if vers == 0 { + vers = maxVersion + } + b.data[1] = byte(vers >> 8) + b.data[2] = byte(vers) + b.data[3] = byte(m >> 8) + b.data[4] = byte(m) + copy(b.data[recordHeaderLen:], data) + c.out.encrypt(b) + _, err = c.conn.Write(b.data) + if err != nil { + break + } + n += m + data = data[m:] + } + c.out.freeBlock(b) + + if typ == recordTypeChangeCipherSpec { + err = c.out.changeCipherSpec() + if err != nil { + // Cannot call sendAlert directly, + // because we already hold c.out.Mutex. + c.tmp[0] = alertLevelError + c.tmp[1] = byte(err.(alert)) + c.writeRecord(recordTypeAlert, c.tmp[0:2]) + c.err = &net.OpError{Op: "local error", Error: err} + return n, c.err + } + } + return +} + +// readHandshake reads the next handshake message from +// the record layer. +// c.in.Mutex < L; c.out.Mutex < L. +func (c *Conn) readHandshake() (interface{}, os.Error) { + for c.hand.Len() < 4 { + if c.err != nil { + return nil, c.err + } + c.readRecord(recordTypeHandshake) + } + + data := c.hand.Bytes() + n := int(data[1])<<16 | int(data[2])<<8 | int(data[3]) + if n > maxHandshake { + c.sendAlert(alertInternalError) + return nil, c.err + } + for c.hand.Len() < 4+n { + if c.err != nil { + return nil, c.err + } + c.readRecord(recordTypeHandshake) + } + data = c.hand.Next(4 + n) + var m handshakeMessage + switch data[0] { + case typeClientHello: + m = new(clientHelloMsg) + case typeServerHello: + m = new(serverHelloMsg) + case typeCertificate: + m = new(certificateMsg) + case typeCertificateRequest: + m = new(certificateRequestMsg) + case typeCertificateStatus: + m = new(certificateStatusMsg) + case typeServerKeyExchange: + m = new(serverKeyExchangeMsg) + case typeServerHelloDone: + m = new(serverHelloDoneMsg) + case typeClientKeyExchange: + m = new(clientKeyExchangeMsg) + case typeCertificateVerify: + m = new(certificateVerifyMsg) + case typeNextProtocol: + m = new(nextProtoMsg) + case typeFinished: + m = new(finishedMsg) + default: + c.sendAlert(alertUnexpectedMessage) + return nil, alertUnexpectedMessage + } + + // The handshake message unmarshallers + // expect to be able to keep references to data, + // so pass in a fresh copy that won't be overwritten. + data = append([]byte(nil), data...) + + if !m.unmarshal(data) { + c.sendAlert(alertUnexpectedMessage) + return nil, alertUnexpectedMessage + } + return m, nil +} + +// Write writes data to the connection. +func (c *Conn) Write(b []byte) (n int, err os.Error) { + if err = c.Handshake(); err != nil { + return + } + + c.out.Lock() + defer c.out.Unlock() + + if !c.handshakeComplete { + return 0, alertInternalError + } + if c.err != nil { + return 0, c.err + } + return c.writeRecord(recordTypeApplicationData, b) +} + +// Read can be made to time out and return err == os.EAGAIN +// after a fixed time limit; see SetTimeout and SetReadTimeout. +func (c *Conn) Read(b []byte) (n int, err os.Error) { + if err = c.Handshake(); err != nil { + return + } + + c.in.Lock() + defer c.in.Unlock() + + for c.input == nil && c.err == nil { + if err := c.readRecord(recordTypeApplicationData); err != nil { + // Soft error, like EAGAIN + return 0, err + } + } + if c.err != nil { + return 0, c.err + } + n, err = c.input.Read(b) + if c.input.off >= len(c.input.data) { + c.in.freeBlock(c.input) + c.input = nil + } + return n, nil +} + +// Close closes the connection. +func (c *Conn) Close() os.Error { + if err := c.Handshake(); err != nil { + return err + } + return c.sendAlert(alertCloseNotify) +} + +// Handshake runs the client or server handshake +// protocol if it has not yet been run. +// Most uses of this package need not call Handshake +// explicitly: the first Read or Write will call it automatically. +func (c *Conn) Handshake() os.Error { + c.handshakeMutex.Lock() + defer c.handshakeMutex.Unlock() + if err := c.error(); err != nil { + return err + } + if c.handshakeComplete { + return nil + } + if c.isClient { + return c.clientHandshake() + } + return c.serverHandshake() +} + +// ConnectionState returns basic TLS details about the connection. +func (c *Conn) ConnectionState() ConnectionState { + c.handshakeMutex.Lock() + defer c.handshakeMutex.Unlock() + + var state ConnectionState + state.HandshakeComplete = c.handshakeComplete + if c.handshakeComplete { + state.NegotiatedProtocol = c.clientProtocol + state.CipherSuite = c.cipherSuite + } + + return state +} + +// OCSPResponse returns the stapled OCSP response from the TLS server, if +// any. (Only valid for client connections.) +func (c *Conn) OCSPResponse() []byte { + c.handshakeMutex.Lock() + defer c.handshakeMutex.Unlock() + + return c.ocspResponse +} + +// PeerCertificates returns the certificate chain that was presented by the +// other side. +func (c *Conn) PeerCertificates() []*x509.Certificate { + c.handshakeMutex.Lock() + defer c.handshakeMutex.Unlock() + + return c.peerCertificates +} + +// VerifyHostname checks that the peer certificate chain is valid for +// connecting to host. If so, it returns nil; if not, it returns an os.Error +// describing the problem. +func (c *Conn) VerifyHostname(host string) os.Error { + c.handshakeMutex.Lock() + defer c.handshakeMutex.Unlock() + if !c.isClient { + return os.ErrorString("VerifyHostname called on TLS server connection") + } + if !c.handshakeComplete { + return os.ErrorString("TLS handshake has not yet been performed") + } + return c.peerCertificates[0].VerifyHostname(host) +} diff --git a/libgo/go/crypto/tls/conn_test.go b/libgo/go/crypto/tls/conn_test.go new file mode 100644 index 000000000..f44a50bed --- /dev/null +++ b/libgo/go/crypto/tls/conn_test.go @@ -0,0 +1,52 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tls + +import ( + "testing" +) + +func TestRoundUp(t *testing.T) { + if roundUp(0, 16) != 0 || + roundUp(1, 16) != 16 || + roundUp(15, 16) != 16 || + roundUp(16, 16) != 16 || + roundUp(17, 16) != 32 { + t.Error("roundUp broken") + } +} + +var paddingTests = []struct { + in []byte + good bool + expectedLen int +}{ + {[]byte{1, 2, 3, 4, 0}, true, 4}, + {[]byte{1, 2, 3, 4, 0, 1}, false, 0}, + {[]byte{1, 2, 3, 4, 99, 99}, false, 0}, + {[]byte{1, 2, 3, 4, 1, 1}, true, 4}, + {[]byte{1, 2, 3, 2, 2, 2}, true, 3}, + {[]byte{1, 2, 3, 3, 3, 3}, true, 2}, + {[]byte{1, 2, 3, 4, 3, 3}, false, 0}, + {[]byte{1, 4, 4, 4, 4, 4}, true, 1}, + {[]byte{5, 5, 5, 5, 5, 5}, true, 0}, + {[]byte{6, 6, 6, 6, 6, 6}, false, 0}, +} + +func TestRemovePadding(t *testing.T) { + for i, test := range paddingTests { + payload, good := removePadding(test.in) + expectedGood := byte(255) + if !test.good { + expectedGood = 0 + } + if good != expectedGood { + t.Errorf("#%d: wrong validity, want:%d got:%d", i, expectedGood, good) + } + if good == 255 && len(payload) != test.expectedLen { + t.Errorf("#%d: got %d, want %d", i, len(payload), test.expectedLen) + } + } +} diff --git a/libgo/go/crypto/tls/generate_cert.go b/libgo/go/crypto/tls/generate_cert.go new file mode 100644 index 000000000..3e0c63938 --- /dev/null +++ b/libgo/go/crypto/tls/generate_cert.go @@ -0,0 +1,70 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Generate a self-signed X.509 certificate for a TLS server. Outputs to +// 'cert.pem' and 'key.pem' and will overwrite existing files. + +package main + +import ( + "crypto/rsa" + "crypto/rand" + "crypto/x509" + "encoding/pem" + "flag" + "log" + "os" + "time" +) + +var hostName *string = flag.String("host", "127.0.0.1", "Hostname to generate a certificate for") + +func main() { + flag.Parse() + + priv, err := rsa.GenerateKey(rand.Reader, 1024) + if err != nil { + log.Exitf("failed to generate private key: %s", err) + return + } + + now := time.Seconds() + + template := x509.Certificate{ + SerialNumber: []byte{0}, + Subject: x509.Name{ + CommonName: *hostName, + Organization: []string{"Acme Co"}, + }, + NotBefore: time.SecondsToUTC(now - 300), + NotAfter: time.SecondsToUTC(now + 60*60*24*365), // valid for 1 year. + + SubjectKeyId: []byte{1, 2, 3, 4}, + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + } + + derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv) + if err != nil { + log.Exitf("Failed to create certificate: %s", err) + return + } + + certOut, err := os.Open("cert.pem", os.O_WRONLY|os.O_CREAT, 0644) + if err != nil { + log.Exitf("failed to open cert.pem for writing: %s", err) + return + } + pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) + certOut.Close() + log.Print("written cert.pem\n") + + keyOut, err := os.Open("key.pem", os.O_WRONLY|os.O_CREAT, 0600) + if err != nil { + log.Print("failed to open key.pem for writing:", err) + return + } + pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)}) + keyOut.Close() + log.Print("written key.pem\n") +} diff --git a/libgo/go/crypto/tls/handshake_client.go b/libgo/go/crypto/tls/handshake_client.go new file mode 100644 index 000000000..1ca33f59d --- /dev/null +++ b/libgo/go/crypto/tls/handshake_client.go @@ -0,0 +1,300 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tls + +import ( + "crypto/rsa" + "crypto/subtle" + "crypto/x509" + "io" + "os" +) + +func (c *Conn) clientHandshake() os.Error { + finishedHash := newFinishedHash() + + if c.config == nil { + c.config = defaultConfig() + } + + hello := &clientHelloMsg{ + vers: maxVersion, + cipherSuites: c.config.cipherSuites(), + compressionMethods: []uint8{compressionNone}, + random: make([]byte, 32), + ocspStapling: true, + serverName: c.config.ServerName, + supportedCurves: []uint16{curveP256, curveP384, curveP521}, + supportedPoints: []uint8{pointFormatUncompressed}, + } + + t := uint32(c.config.time()) + hello.random[0] = byte(t >> 24) + hello.random[1] = byte(t >> 16) + hello.random[2] = byte(t >> 8) + hello.random[3] = byte(t) + _, err := io.ReadFull(c.config.rand(), hello.random[4:]) + if err != nil { + c.sendAlert(alertInternalError) + return os.ErrorString("short read from Rand") + } + + finishedHash.Write(hello.marshal()) + c.writeRecord(recordTypeHandshake, hello.marshal()) + + msg, err := c.readHandshake() + if err != nil { + return err + } + serverHello, ok := msg.(*serverHelloMsg) + if !ok { + return c.sendAlert(alertUnexpectedMessage) + } + finishedHash.Write(serverHello.marshal()) + + vers, ok := mutualVersion(serverHello.vers) + if !ok { + c.sendAlert(alertProtocolVersion) + } + c.vers = vers + c.haveVers = true + + if serverHello.compressionMethod != compressionNone { + return c.sendAlert(alertUnexpectedMessage) + } + + suite, suiteId := mutualCipherSuite(c.config.cipherSuites(), serverHello.cipherSuite) + if suite == nil { + return c.sendAlert(alertHandshakeFailure) + } + + msg, err = c.readHandshake() + if err != nil { + return err + } + certMsg, ok := msg.(*certificateMsg) + if !ok || len(certMsg.certificates) == 0 { + return c.sendAlert(alertUnexpectedMessage) + } + finishedHash.Write(certMsg.marshal()) + + certs := make([]*x509.Certificate, len(certMsg.certificates)) + chain := NewCASet() + for i, asn1Data := range certMsg.certificates { + cert, err := x509.ParseCertificate(asn1Data) + if err != nil { + c.sendAlert(alertBadCertificate) + return os.ErrorString("failed to parse certificate from server: " + err.String()) + } + certs[i] = cert + chain.AddCert(cert) + } + + // If we don't have a root CA set configured then anything is accepted. + // TODO(rsc): Find certificates for OS X 10.6. + for cur := certs[0]; c.config.RootCAs != nil; { + parent := c.config.RootCAs.FindVerifiedParent(cur) + if parent != nil { + break + } + + parent = chain.FindVerifiedParent(cur) + if parent == nil { + c.sendAlert(alertBadCertificate) + return os.ErrorString("could not find root certificate for chain") + } + + if !parent.BasicConstraintsValid || !parent.IsCA { + c.sendAlert(alertBadCertificate) + return os.ErrorString("intermediate certificate does not have CA bit set") + } + // KeyUsage status flags are ignored. From Engineering + // Security, Peter Gutmann: A European government CA marked its + // signing certificates as being valid for encryption only, but + // no-one noticed. Another European CA marked its signature + // keys as not being valid for signatures. A different CA + // marked its own trusted root certificate as being invalid for + // certificate signing. Another national CA distributed a + // certificate to be used to encrypt data for the country’s tax + // authority that was marked as only being usable for digital + // signatures but not for encryption. Yet another CA reversed + // the order of the bit flags in the keyUsage due to confusion + // over encoding endianness, essentially setting a random + // keyUsage in certificates that it issued. Another CA created + // a self-invalidating certificate by adding a certificate + // policy statement stipulating that the certificate had to be + // used strictly as specified in the keyUsage, and a keyUsage + // containing a flag indicating that the RSA encryption key + // could only be used for Diffie-Hellman key agreement. + + cur = parent + } + + if _, ok := certs[0].PublicKey.(*rsa.PublicKey); !ok { + return c.sendAlert(alertUnsupportedCertificate) + } + + c.peerCertificates = certs + + if serverHello.certStatus { + msg, err = c.readHandshake() + if err != nil { + return err + } + cs, ok := msg.(*certificateStatusMsg) + if !ok { + return c.sendAlert(alertUnexpectedMessage) + } + finishedHash.Write(cs.marshal()) + + if cs.statusType == statusTypeOCSP { + c.ocspResponse = cs.response + } + } + + msg, err = c.readHandshake() + if err != nil { + return err + } + + keyAgreement := suite.ka() + + skx, ok := msg.(*serverKeyExchangeMsg) + if ok { + finishedHash.Write(skx.marshal()) + err = keyAgreement.processServerKeyExchange(c.config, hello, serverHello, certs[0], skx) + if err != nil { + c.sendAlert(alertUnexpectedMessage) + return err + } + + msg, err = c.readHandshake() + if err != nil { + return err + } + } + + transmitCert := false + certReq, ok := msg.(*certificateRequestMsg) + if ok { + // We only accept certificates with RSA keys. + rsaAvail := false + for _, certType := range certReq.certificateTypes { + if certType == certTypeRSASign { + rsaAvail = true + break + } + } + + // For now, only send a certificate back if the server gives us an + // empty list of certificateAuthorities. + // + // RFC 4346 on the certificateAuthorities field: + // A list of the distinguished names of acceptable certificate + // authorities. These distinguished names may specify a desired + // distinguished name for a root CA or for a subordinate CA; thus, + // this message can be used to describe both known roots and a + // desired authorization space. If the certificate_authorities + // list is empty then the client MAY send any certificate of the + // appropriate ClientCertificateType, unless there is some + // external arrangement to the contrary. + if rsaAvail && len(certReq.certificateAuthorities) == 0 { + transmitCert = true + } + + finishedHash.Write(certReq.marshal()) + + msg, err = c.readHandshake() + if err != nil { + return err + } + } + + shd, ok := msg.(*serverHelloDoneMsg) + if !ok { + return c.sendAlert(alertUnexpectedMessage) + } + finishedHash.Write(shd.marshal()) + + var cert *x509.Certificate + if transmitCert { + certMsg = new(certificateMsg) + if len(c.config.Certificates) > 0 { + cert, err = x509.ParseCertificate(c.config.Certificates[0].Certificate[0]) + if err == nil && cert.PublicKeyAlgorithm == x509.RSA { + certMsg.certificates = c.config.Certificates[0].Certificate + } else { + cert = nil + } + } + finishedHash.Write(certMsg.marshal()) + c.writeRecord(recordTypeHandshake, certMsg.marshal()) + } + + preMasterSecret, ckx, err := keyAgreement.generateClientKeyExchange(c.config, hello, certs[0]) + if err != nil { + c.sendAlert(alertInternalError) + return err + } + if ckx != nil { + finishedHash.Write(ckx.marshal()) + c.writeRecord(recordTypeHandshake, ckx.marshal()) + } + + if cert != nil { + certVerify := new(certificateVerifyMsg) + var digest [36]byte + copy(digest[0:16], finishedHash.serverMD5.Sum()) + copy(digest[16:36], finishedHash.serverSHA1.Sum()) + signed, err := rsa.SignPKCS1v15(c.config.rand(), c.config.Certificates[0].PrivateKey, rsa.HashMD5SHA1, digest[0:]) + if err != nil { + return c.sendAlert(alertInternalError) + } + certVerify.signature = signed + + finishedHash.Write(certVerify.marshal()) + c.writeRecord(recordTypeHandshake, certVerify.marshal()) + } + + masterSecret, clientMAC, serverMAC, clientKey, serverKey, clientIV, serverIV := + keysFromPreMasterSecret10(preMasterSecret, hello.random, serverHello.random, suite.macLen, suite.keyLen, suite.ivLen) + + clientCipher := suite.cipher(clientKey, clientIV, false /* not for reading */ ) + clientHash := suite.mac(clientMAC) + c.out.prepareCipherSpec(clientCipher, clientHash) + c.writeRecord(recordTypeChangeCipherSpec, []byte{1}) + + finished := new(finishedMsg) + finished.verifyData = finishedHash.clientSum(masterSecret) + finishedHash.Write(finished.marshal()) + c.writeRecord(recordTypeHandshake, finished.marshal()) + + serverCipher := suite.cipher(serverKey, serverIV, true /* for reading */ ) + serverHash := suite.mac(serverMAC) + c.in.prepareCipherSpec(serverCipher, serverHash) + c.readRecord(recordTypeChangeCipherSpec) + if c.err != nil { + return c.err + } + + msg, err = c.readHandshake() + if err != nil { + return err + } + serverFinished, ok := msg.(*finishedMsg) + if !ok { + return c.sendAlert(alertUnexpectedMessage) + } + + verify := finishedHash.serverSum(masterSecret) + if len(verify) != len(serverFinished.verifyData) || + subtle.ConstantTimeCompare(verify, serverFinished.verifyData) != 1 { + return c.sendAlert(alertHandshakeFailure) + } + + c.handshakeComplete = true + c.cipherSuite = suiteId + return nil +} diff --git a/libgo/go/crypto/tls/handshake_client_test.go b/libgo/go/crypto/tls/handshake_client_test.go new file mode 100644 index 000000000..e5c9684b9 --- /dev/null +++ b/libgo/go/crypto/tls/handshake_client_test.go @@ -0,0 +1,211 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tls + +import ( + "bytes" + "flag" + "io" + "net" + "testing" +) + +func testClientScript(t *testing.T, name string, clientScript [][]byte, config *Config) { + c, s := net.Pipe() + cli := Client(c, config) + go func() { + cli.Write([]byte("hello\n")) + cli.Close() + }() + + defer c.Close() + for i, b := range clientScript { + if i%2 == 1 { + s.Write(b) + continue + } + bb := make([]byte, len(b)) + _, err := io.ReadFull(s, bb) + if err != nil { + t.Fatalf("%s #%d: %s", name, i, err) + } + if !bytes.Equal(b, bb) { + t.Fatalf("%s #%d: mismatch on read: got:%x want:%x", name, i, bb, b) + } + } +} + +func TestHandshakeClientRC4(t *testing.T) { + testClientScript(t, "RC4", rc4ClientScript, testConfig) +} + +var connect = flag.Bool("connect", false, "connect to a TLS server on :10443") + +func TestRunClient(t *testing.T) { + if !*connect { + return + } + + testConfig.CipherSuites = []uint16{TLS_ECDHE_RSA_WITH_RC4_128_SHA} + + conn, err := Dial("tcp", "", "127.0.0.1:10443", testConfig) + if err != nil { + t.Fatal(err) + } + + conn.Write([]byte("hello\n")) + conn.Close() +} + +// Script of interaction with gnutls implementation. +// The values for this test are obtained by building and running in client mode: +// % gotest -match "TestRunClient" -connect +// and then: +// % gnutls-serv -p 10443 --debug 100 --x509keyfile key.pem --x509certfile cert.pem -a > /tmp/log 2>&1 +// % python parse-gnutls-cli-debug-log.py < /tmp/log +// +// Where key.pem is: +// -----BEGIN RSA PRIVATE KEY----- +// MIIBPAIBAAJBAJ+zw4Qnlf8SMVIPFe9GEcStgOY2Ww/dgNdhjeD8ckUJNP5VZkVD +// TGiXav6ooKXfX3j/7tdkuD8Ey2//Kv7+ue0CAwEAAQJAN6W31vDEP2DjdqhzCDDu +// OA4NACqoiFqyblo7yc2tM4h4xMbC3Yx5UKMN9ZkCtX0gzrz6DyF47bdKcWBzNWCj +// gQIhANEoojVt7hq+SQ6MCN6FTAysGgQf56Q3TYoJMoWvdiXVAiEAw3e3rc+VJpOz +// rHuDo6bgpjUAAXM+v3fcpsfZSNO6V7kCIQCtbVjanpUwvZkMI9by02oUk9taki3b +// PzPfAfNPYAbCJQIhAJXNQDWyqwn/lGmR11cqY2y9nZ1+5w3yHGatLrcDnQHxAiEA +// vnlEGo8K85u+KwIOimM48ZG8oTk7iFdkqLJR1utT3aU= +// -----END RSA PRIVATE KEY----- +// +// and cert.pem is: +// -----BEGIN CERTIFICATE----- +// MIIBoDCCAUoCAQAwDQYJKoZIhvcNAQEEBQAwYzELMAkGA1UEBhMCQVUxEzARBgNV +// BAgTClF1ZWVuc2xhbmQxGjAYBgNVBAoTEUNyeXB0U29mdCBQdHkgTHRkMSMwIQYD +// VQQDExpTZXJ2ZXIgdGVzdCBjZXJ0ICg1MTIgYml0KTAeFw05NzA5MDkwMzQxMjZa +// Fw05NzEwMDkwMzQxMjZaMF4xCzAJBgNVBAYTAkFVMRMwEQYDVQQIEwpTb21lLVN0 +// YXRlMSEwHwYDVQQKExhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQxFzAVBgNVBAMT +// DkVyaWMgdGhlIFlvdW5nMFEwCQYFKw4DAgwFAANEAAJBALVEqPODnpI4rShlY8S7 +// tB713JNvabvn6Gned7zylwLLiXQAo/PAT6mfdWPTyCX9RlId/Aroh1ou893BA32Q +// sggwDQYJKoZIhvcNAQEEBQADQQCU5SSgapJSdRXJoX+CpCvFy+JVh9HpSjCpSNKO +// 19raHv98hKAUJuP9HyM+SUsffO6mAIgitUaqW8/wDMePhEC3 +// -----END CERTIFICATE----- +var rc4ClientScript = [][]byte{ + { + 0x16, 0x03, 0x01, 0x00, 0x4a, 0x01, 0x00, 0x00, + 0x46, 0x03, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x05, + 0x01, 0x00, 0x00, 0x1b, 0x00, 0x05, 0x00, 0x05, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a, 0x00, + 0x08, 0x00, 0x06, 0x00, 0x17, 0x00, 0x18, 0x00, + 0x19, 0x00, 0x0b, 0x00, 0x02, 0x01, 0x00, + }, + + { + 0x16, 0x03, 0x01, 0x00, 0x4a, 0x02, 0x00, 0x00, + 0x46, 0x03, 0x01, 0x4d, 0x0a, 0x56, 0x16, 0xb5, + 0x91, 0xd1, 0xcb, 0x80, 0x4d, 0xc7, 0x46, 0xf3, + 0x37, 0x0c, 0xef, 0xea, 0x64, 0x11, 0x14, 0x56, + 0x97, 0x9b, 0xc5, 0x67, 0x08, 0xb7, 0x13, 0xea, + 0xf8, 0xc9, 0xb3, 0x20, 0xe2, 0xfc, 0x41, 0xf6, + 0x96, 0x90, 0x9d, 0x43, 0x9b, 0xe9, 0x6e, 0xf8, + 0x41, 0x16, 0xcc, 0xf3, 0xc7, 0xde, 0xda, 0x5a, + 0xa1, 0x33, 0x69, 0xe2, 0xde, 0x5b, 0xaf, 0x2a, + 0x92, 0xe7, 0xd4, 0xa0, 0x00, 0x05, 0x00, 0x16, + 0x03, 0x01, 0x01, 0xf7, 0x0b, 0x00, 0x01, 0xf3, + 0x00, 0x01, 0xf0, 0x00, 0x01, 0xed, 0x30, 0x82, + 0x01, 0xe9, 0x30, 0x82, 0x01, 0x52, 0x02, 0x01, + 0x06, 0x30, 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, + 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x04, 0x05, 0x00, + 0x30, 0x5b, 0x31, 0x0b, 0x30, 0x09, 0x06, 0x03, + 0x55, 0x04, 0x06, 0x13, 0x02, 0x41, 0x55, 0x31, + 0x13, 0x30, 0x11, 0x06, 0x03, 0x55, 0x04, 0x08, + 0x13, 0x0a, 0x51, 0x75, 0x65, 0x65, 0x6e, 0x73, + 0x6c, 0x61, 0x6e, 0x64, 0x31, 0x1a, 0x30, 0x18, + 0x06, 0x03, 0x55, 0x04, 0x0a, 0x13, 0x11, 0x43, + 0x72, 0x79, 0x70, 0x74, 0x53, 0x6f, 0x66, 0x74, + 0x20, 0x50, 0x74, 0x79, 0x20, 0x4c, 0x74, 0x64, + 0x31, 0x1b, 0x30, 0x19, 0x06, 0x03, 0x55, 0x04, + 0x03, 0x13, 0x12, 0x54, 0x65, 0x73, 0x74, 0x20, + 0x43, 0x41, 0x20, 0x28, 0x31, 0x30, 0x32, 0x34, + 0x20, 0x62, 0x69, 0x74, 0x29, 0x30, 0x1e, 0x17, + 0x0d, 0x30, 0x30, 0x31, 0x30, 0x31, 0x36, 0x32, + 0x32, 0x33, 0x31, 0x30, 0x33, 0x5a, 0x17, 0x0d, + 0x30, 0x33, 0x30, 0x31, 0x31, 0x34, 0x32, 0x32, + 0x33, 0x31, 0x30, 0x33, 0x5a, 0x30, 0x63, 0x31, + 0x0b, 0x30, 0x09, 0x06, 0x03, 0x55, 0x04, 0x06, + 0x13, 0x02, 0x41, 0x55, 0x31, 0x13, 0x30, 0x11, + 0x06, 0x03, 0x55, 0x04, 0x08, 0x13, 0x0a, 0x51, + 0x75, 0x65, 0x65, 0x6e, 0x73, 0x6c, 0x61, 0x6e, + 0x64, 0x31, 0x1a, 0x30, 0x18, 0x06, 0x03, 0x55, + 0x04, 0x0a, 0x13, 0x11, 0x43, 0x72, 0x79, 0x70, + 0x74, 0x53, 0x6f, 0x66, 0x74, 0x20, 0x50, 0x74, + 0x79, 0x20, 0x4c, 0x74, 0x64, 0x31, 0x23, 0x30, + 0x21, 0x06, 0x03, 0x55, 0x04, 0x03, 0x13, 0x1a, + 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x20, 0x74, + 0x65, 0x73, 0x74, 0x20, 0x63, 0x65, 0x72, 0x74, + 0x20, 0x28, 0x35, 0x31, 0x32, 0x20, 0x62, 0x69, + 0x74, 0x29, 0x30, 0x5c, 0x30, 0x0d, 0x06, 0x09, + 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, + 0x01, 0x05, 0x00, 0x03, 0x4b, 0x00, 0x30, 0x48, + 0x02, 0x41, 0x00, 0x9f, 0xb3, 0xc3, 0x84, 0x27, + 0x95, 0xff, 0x12, 0x31, 0x52, 0x0f, 0x15, 0xef, + 0x46, 0x11, 0xc4, 0xad, 0x80, 0xe6, 0x36, 0x5b, + 0x0f, 0xdd, 0x80, 0xd7, 0x61, 0x8d, 0xe0, 0xfc, + 0x72, 0x45, 0x09, 0x34, 0xfe, 0x55, 0x66, 0x45, + 0x43, 0x4c, 0x68, 0x97, 0x6a, 0xfe, 0xa8, 0xa0, + 0xa5, 0xdf, 0x5f, 0x78, 0xff, 0xee, 0xd7, 0x64, + 0xb8, 0x3f, 0x04, 0xcb, 0x6f, 0xff, 0x2a, 0xfe, + 0xfe, 0xb9, 0xed, 0x02, 0x03, 0x01, 0x00, 0x01, + 0x30, 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, + 0xf7, 0x0d, 0x01, 0x01, 0x04, 0x05, 0x00, 0x03, + 0x81, 0x81, 0x00, 0x93, 0xd2, 0x0a, 0xc5, 0x41, + 0xe6, 0x5a, 0xa9, 0x86, 0xf9, 0x11, 0x87, 0xe4, + 0xdb, 0x45, 0xe2, 0xc5, 0x95, 0x78, 0x1a, 0x6c, + 0x80, 0x6d, 0x73, 0x1f, 0xb4, 0x6d, 0x44, 0xa3, + 0xba, 0x86, 0x88, 0xc8, 0x58, 0xcd, 0x1c, 0x06, + 0x35, 0x6c, 0x44, 0x62, 0x88, 0xdf, 0xe4, 0xf6, + 0x64, 0x61, 0x95, 0xef, 0x4a, 0xa6, 0x7f, 0x65, + 0x71, 0xd7, 0x6b, 0x88, 0x39, 0xf6, 0x32, 0xbf, + 0xac, 0x93, 0x67, 0x69, 0x51, 0x8c, 0x93, 0xec, + 0x48, 0x5f, 0xc9, 0xb1, 0x42, 0xf9, 0x55, 0xd2, + 0x7e, 0x4e, 0xf4, 0xf2, 0x21, 0x6b, 0x90, 0x57, + 0xe6, 0xd7, 0x99, 0x9e, 0x41, 0xca, 0x80, 0xbf, + 0x1a, 0x28, 0xa2, 0xca, 0x5b, 0x50, 0x4a, 0xed, + 0x84, 0xe7, 0x82, 0xc7, 0xd2, 0xcf, 0x36, 0x9e, + 0x6a, 0x67, 0xb9, 0x88, 0xa7, 0xf3, 0x8a, 0xd0, + 0x04, 0xf8, 0xe8, 0xc6, 0x17, 0xe3, 0xc5, 0x29, + 0xbc, 0x17, 0xf1, 0x16, 0x03, 0x01, 0x00, 0x04, + 0x0e, 0x00, 0x00, 0x00, + }, + + { + 0x16, 0x03, 0x01, 0x00, 0x46, 0x10, 0x00, 0x00, + 0x42, 0x00, 0x40, 0x87, 0xa1, 0x1f, 0x14, 0xe1, + 0xfb, 0x91, 0xac, 0x58, 0x2e, 0xf3, 0x71, 0xce, + 0x01, 0x85, 0x2c, 0xc7, 0xfe, 0x84, 0x87, 0x82, + 0xb7, 0x57, 0xdb, 0x37, 0x4d, 0x46, 0x83, 0x67, + 0x52, 0x82, 0x51, 0x01, 0x95, 0x23, 0x68, 0x69, + 0x6b, 0xd0, 0xa7, 0xa7, 0xe5, 0x88, 0xd0, 0x47, + 0x71, 0xb8, 0xd2, 0x03, 0x05, 0x25, 0x56, 0x5c, + 0x10, 0x08, 0xc6, 0x9b, 0xd4, 0x67, 0xcd, 0x28, + 0xbe, 0x9c, 0x48, 0x14, 0x03, 0x01, 0x00, 0x01, + 0x01, 0x16, 0x03, 0x01, 0x00, 0x24, 0xc1, 0xb8, + 0xd3, 0x7f, 0xc5, 0xc2, 0x5a, 0x1d, 0x6d, 0x5b, + 0x2d, 0x5c, 0x82, 0x87, 0xc2, 0x6f, 0x0d, 0x63, + 0x7b, 0x72, 0x2b, 0xda, 0x69, 0xc4, 0xfe, 0x3c, + 0x84, 0xa1, 0x5a, 0x62, 0x38, 0x37, 0xc6, 0x54, + 0x25, 0x2a, + }, + + { + 0x14, 0x03, 0x01, 0x00, 0x01, 0x01, 0x16, 0x03, + 0x01, 0x00, 0x24, 0xea, 0x88, 0x9c, 0x00, 0xf6, + 0x35, 0xb8, 0x42, 0x7f, 0x15, 0x17, 0x76, 0x5e, + 0x4b, 0x24, 0xcb, 0x7e, 0xa0, 0x7b, 0xc3, 0x70, + 0x52, 0x0a, 0x88, 0x2a, 0x7a, 0x45, 0x59, 0x90, + 0x59, 0xac, 0xc6, 0xb5, 0x56, 0x55, 0x96, + }, +} diff --git a/libgo/go/crypto/tls/handshake_messages.go b/libgo/go/crypto/tls/handshake_messages.go new file mode 100644 index 000000000..e5e856271 --- /dev/null +++ b/libgo/go/crypto/tls/handshake_messages.go @@ -0,0 +1,904 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tls + +type clientHelloMsg struct { + raw []byte + vers uint16 + random []byte + sessionId []byte + cipherSuites []uint16 + compressionMethods []uint8 + nextProtoNeg bool + serverName string + ocspStapling bool + supportedCurves []uint16 + supportedPoints []uint8 +} + +func (m *clientHelloMsg) marshal() []byte { + if m.raw != nil { + return m.raw + } + + length := 2 + 32 + 1 + len(m.sessionId) + 2 + len(m.cipherSuites)*2 + 1 + len(m.compressionMethods) + numExtensions := 0 + extensionsLength := 0 + if m.nextProtoNeg { + numExtensions++ + } + if m.ocspStapling { + extensionsLength += 1 + 2 + 2 + numExtensions++ + } + if len(m.serverName) > 0 { + extensionsLength += 5 + len(m.serverName) + numExtensions++ + } + if len(m.supportedCurves) > 0 { + extensionsLength += 2 + 2*len(m.supportedCurves) + numExtensions++ + } + if len(m.supportedPoints) > 0 { + extensionsLength += 1 + len(m.supportedPoints) + numExtensions++ + } + if numExtensions > 0 { + extensionsLength += 4 * numExtensions + length += 2 + extensionsLength + } + + x := make([]byte, 4+length) + x[0] = typeClientHello + x[1] = uint8(length >> 16) + x[2] = uint8(length >> 8) + x[3] = uint8(length) + x[4] = uint8(m.vers >> 8) + x[5] = uint8(m.vers) + copy(x[6:38], m.random) + x[38] = uint8(len(m.sessionId)) + copy(x[39:39+len(m.sessionId)], m.sessionId) + y := x[39+len(m.sessionId):] + y[0] = uint8(len(m.cipherSuites) >> 7) + y[1] = uint8(len(m.cipherSuites) << 1) + for i, suite := range m.cipherSuites { + y[2+i*2] = uint8(suite >> 8) + y[3+i*2] = uint8(suite) + } + z := y[2+len(m.cipherSuites)*2:] + z[0] = uint8(len(m.compressionMethods)) + copy(z[1:], m.compressionMethods) + + z = z[1+len(m.compressionMethods):] + if numExtensions > 0 { + z[0] = byte(extensionsLength >> 8) + z[1] = byte(extensionsLength) + z = z[2:] + } + if m.nextProtoNeg { + z[0] = byte(extensionNextProtoNeg >> 8) + z[1] = byte(extensionNextProtoNeg) + // The length is always 0 + z = z[4:] + } + if len(m.serverName) > 0 { + z[0] = byte(extensionServerName >> 8) + z[1] = byte(extensionServerName) + l := len(m.serverName) + 5 + z[2] = byte(l >> 8) + z[3] = byte(l) + z = z[4:] + + // RFC 3546, section 3.1 + // + // struct { + // NameType name_type; + // select (name_type) { + // case host_name: HostName; + // } name; + // } ServerName; + // + // enum { + // host_name(0), (255) + // } NameType; + // + // opaque HostName<1..2^16-1>; + // + // struct { + // ServerName server_name_list<1..2^16-1> + // } ServerNameList; + + z[0] = byte((len(m.serverName) + 3) >> 8) + z[1] = byte(len(m.serverName) + 3) + z[3] = byte(len(m.serverName) >> 8) + z[4] = byte(len(m.serverName)) + copy(z[5:], []byte(m.serverName)) + z = z[l:] + } + if m.ocspStapling { + // RFC 4366, section 3.6 + z[0] = byte(extensionStatusRequest >> 8) + z[1] = byte(extensionStatusRequest) + z[2] = 0 + z[3] = 5 + z[4] = 1 // OCSP type + // Two zero valued uint16s for the two lengths. + z = z[9:] + } + if len(m.supportedCurves) > 0 { + // http://tools.ietf.org/html/rfc4492#section-5.5.1 + z[0] = byte(extensionSupportedCurves >> 8) + z[1] = byte(extensionSupportedCurves) + l := 2 + 2*len(m.supportedCurves) + z[2] = byte(l >> 8) + z[3] = byte(l) + l -= 2 + z[4] = byte(l >> 8) + z[5] = byte(l) + z = z[6:] + for _, curve := range m.supportedCurves { + z[0] = byte(curve >> 8) + z[1] = byte(curve) + z = z[2:] + } + } + if len(m.supportedPoints) > 0 { + // http://tools.ietf.org/html/rfc4492#section-5.5.2 + z[0] = byte(extensionSupportedPoints >> 8) + z[1] = byte(extensionSupportedPoints) + l := 1 + len(m.supportedPoints) + z[2] = byte(l >> 8) + z[3] = byte(l) + l-- + z[4] = byte(l) + z = z[5:] + for _, pointFormat := range m.supportedPoints { + z[0] = byte(pointFormat) + z = z[1:] + } + } + + m.raw = x + + return x +} + +func (m *clientHelloMsg) unmarshal(data []byte) bool { + if len(data) < 42 { + return false + } + m.raw = data + m.vers = uint16(data[4])<<8 | uint16(data[5]) + m.random = data[6:38] + sessionIdLen := int(data[38]) + if sessionIdLen > 32 || len(data) < 39+sessionIdLen { + return false + } + m.sessionId = data[39 : 39+sessionIdLen] + data = data[39+sessionIdLen:] + if len(data) < 2 { + return false + } + // cipherSuiteLen is the number of bytes of cipher suite numbers. Since + // they are uint16s, the number must be even. + cipherSuiteLen := int(data[0])<<8 | int(data[1]) + if cipherSuiteLen%2 == 1 || len(data) < 2+cipherSuiteLen { + return false + } + numCipherSuites := cipherSuiteLen / 2 + m.cipherSuites = make([]uint16, numCipherSuites) + for i := 0; i < numCipherSuites; i++ { + m.cipherSuites[i] = uint16(data[2+2*i])<<8 | uint16(data[3+2*i]) + } + data = data[2+cipherSuiteLen:] + if len(data) < 1 { + return false + } + compressionMethodsLen := int(data[0]) + if len(data) < 1+compressionMethodsLen { + return false + } + m.compressionMethods = data[1 : 1+compressionMethodsLen] + + data = data[1+compressionMethodsLen:] + + m.nextProtoNeg = false + m.serverName = "" + m.ocspStapling = false + + if len(data) == 0 { + // ClientHello is optionally followed by extension data + return true + } + if len(data) < 2 { + return false + } + + extensionsLength := int(data[0])<<8 | int(data[1]) + data = data[2:] + if extensionsLength != len(data) { + return false + } + + for len(data) != 0 { + if len(data) < 4 { + return false + } + extension := uint16(data[0])<<8 | uint16(data[1]) + length := int(data[2])<<8 | int(data[3]) + data = data[4:] + if len(data) < length { + return false + } + + switch extension { + case extensionServerName: + if length < 2 { + return false + } + numNames := int(data[0])<<8 | int(data[1]) + d := data[2:] + for i := 0; i < numNames; i++ { + if len(d) < 3 { + return false + } + nameType := d[0] + nameLen := int(d[1])<<8 | int(d[2]) + d = d[3:] + if len(d) < nameLen { + return false + } + if nameType == 0 { + m.serverName = string(d[0:nameLen]) + break + } + d = d[nameLen:] + } + case extensionNextProtoNeg: + if length > 0 { + return false + } + m.nextProtoNeg = true + case extensionStatusRequest: + m.ocspStapling = length > 0 && data[0] == statusTypeOCSP + case extensionSupportedCurves: + // http://tools.ietf.org/html/rfc4492#section-5.5.1 + if length < 2 { + return false + } + l := int(data[0])<<8 | int(data[1]) + if l%2 == 1 || length != l+2 { + return false + } + numCurves := l / 2 + m.supportedCurves = make([]uint16, numCurves) + d := data[2:] + for i := 0; i < numCurves; i++ { + m.supportedCurves[i] = uint16(d[0])<<8 | uint16(d[1]) + d = d[2:] + } + case extensionSupportedPoints: + // http://tools.ietf.org/html/rfc4492#section-5.5.2 + if length < 1 { + return false + } + l := int(data[0]) + if length != l+1 { + return false + } + m.supportedPoints = make([]uint8, l) + copy(m.supportedPoints, data[1:]) + } + data = data[length:] + } + + return true +} + +type serverHelloMsg struct { + raw []byte + vers uint16 + random []byte + sessionId []byte + cipherSuite uint16 + compressionMethod uint8 + nextProtoNeg bool + nextProtos []string + certStatus bool +} + +func (m *serverHelloMsg) marshal() []byte { + if m.raw != nil { + return m.raw + } + + length := 38 + len(m.sessionId) + numExtensions := 0 + extensionsLength := 0 + + nextProtoLen := 0 + if m.nextProtoNeg { + numExtensions++ + for _, v := range m.nextProtos { + nextProtoLen += len(v) + } + nextProtoLen += len(m.nextProtos) + extensionsLength += nextProtoLen + } + if m.certStatus { + numExtensions++ + } + if numExtensions > 0 { + extensionsLength += 4 * numExtensions + length += 2 + extensionsLength + } + + x := make([]byte, 4+length) + x[0] = typeServerHello + x[1] = uint8(length >> 16) + x[2] = uint8(length >> 8) + x[3] = uint8(length) + x[4] = uint8(m.vers >> 8) + x[5] = uint8(m.vers) + copy(x[6:38], m.random) + x[38] = uint8(len(m.sessionId)) + copy(x[39:39+len(m.sessionId)], m.sessionId) + z := x[39+len(m.sessionId):] + z[0] = uint8(m.cipherSuite >> 8) + z[1] = uint8(m.cipherSuite) + z[2] = uint8(m.compressionMethod) + + z = z[3:] + if numExtensions > 0 { + z[0] = byte(extensionsLength >> 8) + z[1] = byte(extensionsLength) + z = z[2:] + } + if m.nextProtoNeg { + z[0] = byte(extensionNextProtoNeg >> 8) + z[1] = byte(extensionNextProtoNeg) + z[2] = byte(nextProtoLen >> 8) + z[3] = byte(nextProtoLen) + z = z[4:] + + for _, v := range m.nextProtos { + l := len(v) + if l > 255 { + l = 255 + } + z[0] = byte(l) + copy(z[1:], []byte(v[0:l])) + z = z[1+l:] + } + } + if m.certStatus { + z[0] = byte(extensionStatusRequest >> 8) + z[1] = byte(extensionStatusRequest) + z = z[4:] + } + + m.raw = x + + return x +} + +func (m *serverHelloMsg) unmarshal(data []byte) bool { + if len(data) < 42 { + return false + } + m.raw = data + m.vers = uint16(data[4])<<8 | uint16(data[5]) + m.random = data[6:38] + sessionIdLen := int(data[38]) + if sessionIdLen > 32 || len(data) < 39+sessionIdLen { + return false + } + m.sessionId = data[39 : 39+sessionIdLen] + data = data[39+sessionIdLen:] + if len(data) < 3 { + return false + } + m.cipherSuite = uint16(data[0])<<8 | uint16(data[1]) + m.compressionMethod = data[2] + data = data[3:] + + m.nextProtoNeg = false + m.nextProtos = nil + m.certStatus = false + + if len(data) == 0 { + // ServerHello is optionally followed by extension data + return true + } + if len(data) < 2 { + return false + } + + extensionsLength := int(data[0])<<8 | int(data[1]) + data = data[2:] + if len(data) != extensionsLength { + return false + } + + for len(data) != 0 { + if len(data) < 4 { + return false + } + extension := uint16(data[0])<<8 | uint16(data[1]) + length := int(data[2])<<8 | int(data[3]) + data = data[4:] + if len(data) < length { + return false + } + + switch extension { + case extensionNextProtoNeg: + m.nextProtoNeg = true + d := data + for len(d) > 0 { + l := int(d[0]) + d = d[1:] + if l == 0 || l > len(d) { + return false + } + m.nextProtos = append(m.nextProtos, string(d[0:l])) + d = d[l:] + } + case extensionStatusRequest: + if length > 0 { + return false + } + m.certStatus = true + } + data = data[length:] + } + + return true +} + +type certificateMsg struct { + raw []byte + certificates [][]byte +} + +func (m *certificateMsg) marshal() (x []byte) { + if m.raw != nil { + return m.raw + } + + var i int + for _, slice := range m.certificates { + i += len(slice) + } + + length := 3 + 3*len(m.certificates) + i + x = make([]byte, 4+length) + x[0] = typeCertificate + x[1] = uint8(length >> 16) + x[2] = uint8(length >> 8) + x[3] = uint8(length) + + certificateOctets := length - 3 + x[4] = uint8(certificateOctets >> 16) + x[5] = uint8(certificateOctets >> 8) + x[6] = uint8(certificateOctets) + + y := x[7:] + for _, slice := range m.certificates { + y[0] = uint8(len(slice) >> 16) + y[1] = uint8(len(slice) >> 8) + y[2] = uint8(len(slice)) + copy(y[3:], slice) + y = y[3+len(slice):] + } + + m.raw = x + return +} + +func (m *certificateMsg) unmarshal(data []byte) bool { + if len(data) < 7 { + return false + } + + m.raw = data + certsLen := uint32(data[4])<<16 | uint32(data[5])<<8 | uint32(data[6]) + if uint32(len(data)) != certsLen+7 { + return false + } + + numCerts := 0 + d := data[7:] + for certsLen > 0 { + if len(d) < 4 { + return false + } + certLen := uint32(d[0])<<24 | uint32(d[1])<<8 | uint32(d[2]) + if uint32(len(d)) < 3+certLen { + return false + } + d = d[3+certLen:] + certsLen -= 3 + certLen + numCerts++ + } + + m.certificates = make([][]byte, numCerts) + d = data[7:] + for i := 0; i < numCerts; i++ { + certLen := uint32(d[0])<<24 | uint32(d[1])<<8 | uint32(d[2]) + m.certificates[i] = d[3 : 3+certLen] + d = d[3+certLen:] + } + + return true +} + +type serverKeyExchangeMsg struct { + raw []byte + key []byte +} + +func (m *serverKeyExchangeMsg) marshal() []byte { + if m.raw != nil { + return m.raw + } + length := len(m.key) + x := make([]byte, length+4) + x[0] = typeServerKeyExchange + x[1] = uint8(length >> 16) + x[2] = uint8(length >> 8) + x[3] = uint8(length) + copy(x[4:], m.key) + + m.raw = x + return x +} + +func (m *serverKeyExchangeMsg) unmarshal(data []byte) bool { + m.raw = data + if len(data) < 4 { + return false + } + m.key = data[4:] + return true +} + +type certificateStatusMsg struct { + raw []byte + statusType uint8 + response []byte +} + +func (m *certificateStatusMsg) marshal() []byte { + if m.raw != nil { + return m.raw + } + + var x []byte + if m.statusType == statusTypeOCSP { + x = make([]byte, 4+4+len(m.response)) + x[0] = typeCertificateStatus + l := len(m.response) + 4 + x[1] = byte(l >> 16) + x[2] = byte(l >> 8) + x[3] = byte(l) + x[4] = statusTypeOCSP + + l -= 4 + x[5] = byte(l >> 16) + x[6] = byte(l >> 8) + x[7] = byte(l) + copy(x[8:], m.response) + } else { + x = []byte{typeCertificateStatus, 0, 0, 1, m.statusType} + } + + m.raw = x + return x +} + +func (m *certificateStatusMsg) unmarshal(data []byte) bool { + m.raw = data + if len(data) < 5 { + return false + } + m.statusType = data[4] + + m.response = nil + if m.statusType == statusTypeOCSP { + if len(data) < 8 { + return false + } + respLen := uint32(data[5])<<16 | uint32(data[6])<<8 | uint32(data[7]) + if uint32(len(data)) != 4+4+respLen { + return false + } + m.response = data[8:] + } + return true +} + +type serverHelloDoneMsg struct{} + +func (m *serverHelloDoneMsg) marshal() []byte { + x := make([]byte, 4) + x[0] = typeServerHelloDone + return x +} + +func (m *serverHelloDoneMsg) unmarshal(data []byte) bool { + return len(data) == 4 +} + +type clientKeyExchangeMsg struct { + raw []byte + ciphertext []byte +} + +func (m *clientKeyExchangeMsg) marshal() []byte { + if m.raw != nil { + return m.raw + } + length := len(m.ciphertext) + x := make([]byte, length+4) + x[0] = typeClientKeyExchange + x[1] = uint8(length >> 16) + x[2] = uint8(length >> 8) + x[3] = uint8(length) + copy(x[4:], m.ciphertext) + + m.raw = x + return x +} + +func (m *clientKeyExchangeMsg) unmarshal(data []byte) bool { + m.raw = data + if len(data) < 4 { + return false + } + l := int(data[1])<<16 | int(data[2])<<8 | int(data[3]) + if l != len(data)-4 { + return false + } + m.ciphertext = data[4:] + return true +} + +type finishedMsg struct { + raw []byte + verifyData []byte +} + +func (m *finishedMsg) marshal() (x []byte) { + if m.raw != nil { + return m.raw + } + + x = make([]byte, 16) + x[0] = typeFinished + x[3] = 12 + copy(x[4:], m.verifyData) + m.raw = x + return +} + +func (m *finishedMsg) unmarshal(data []byte) bool { + m.raw = data + if len(data) != 4+12 { + return false + } + m.verifyData = data[4:] + return true +} + +type nextProtoMsg struct { + raw []byte + proto string +} + +func (m *nextProtoMsg) marshal() []byte { + if m.raw != nil { + return m.raw + } + l := len(m.proto) + if l > 255 { + l = 255 + } + + padding := 32 - (l+2)%32 + length := l + padding + 2 + x := make([]byte, length+4) + x[0] = typeNextProtocol + x[1] = uint8(length >> 16) + x[2] = uint8(length >> 8) + x[3] = uint8(length) + + y := x[4:] + y[0] = byte(l) + copy(y[1:], []byte(m.proto[0:l])) + y = y[1+l:] + y[0] = byte(padding) + + m.raw = x + + return x +} + +func (m *nextProtoMsg) unmarshal(data []byte) bool { + m.raw = data + + if len(data) < 5 { + return false + } + data = data[4:] + protoLen := int(data[0]) + data = data[1:] + if len(data) < protoLen { + return false + } + m.proto = string(data[0:protoLen]) + data = data[protoLen:] + + if len(data) < 1 { + return false + } + paddingLen := int(data[0]) + data = data[1:] + if len(data) != paddingLen { + return false + } + + return true +} + +type certificateRequestMsg struct { + raw []byte + certificateTypes []byte + certificateAuthorities [][]byte +} + +func (m *certificateRequestMsg) marshal() (x []byte) { + if m.raw != nil { + return m.raw + } + + // See http://tools.ietf.org/html/rfc4346#section-7.4.4 + length := 1 + len(m.certificateTypes) + 2 + for _, ca := range m.certificateAuthorities { + length += 2 + len(ca) + } + + x = make([]byte, 4+length) + x[0] = typeCertificateRequest + x[1] = uint8(length >> 16) + x[2] = uint8(length >> 8) + x[3] = uint8(length) + + x[4] = uint8(len(m.certificateTypes)) + + copy(x[5:], m.certificateTypes) + y := x[5+len(m.certificateTypes):] + + numCA := len(m.certificateAuthorities) + y[0] = uint8(numCA >> 8) + y[1] = uint8(numCA) + y = y[2:] + for _, ca := range m.certificateAuthorities { + y[0] = uint8(len(ca) >> 8) + y[1] = uint8(len(ca)) + y = y[2:] + copy(y, ca) + y = y[len(ca):] + } + + m.raw = x + + return +} + +func (m *certificateRequestMsg) unmarshal(data []byte) bool { + m.raw = data + + if len(data) < 5 { + return false + } + + length := uint32(data[1])<<16 | uint32(data[2])<<8 | uint32(data[3]) + if uint32(len(data))-4 != length { + return false + } + + numCertTypes := int(data[4]) + data = data[5:] + if numCertTypes == 0 || len(data) <= numCertTypes { + return false + } + + m.certificateTypes = make([]byte, numCertTypes) + if copy(m.certificateTypes, data) != numCertTypes { + return false + } + + data = data[numCertTypes:] + if len(data) < 2 { + return false + } + + numCAs := uint16(data[0])<<16 | uint16(data[1]) + data = data[2:] + + m.certificateAuthorities = make([][]byte, numCAs) + for i := uint16(0); i < numCAs; i++ { + if len(data) < 2 { + return false + } + caLen := uint16(data[0])<<16 | uint16(data[1]) + + data = data[2:] + if len(data) < int(caLen) { + return false + } + + ca := make([]byte, caLen) + copy(ca, data) + m.certificateAuthorities[i] = ca + data = data[caLen:] + } + + if len(data) > 0 { + return false + } + + return true +} + +type certificateVerifyMsg struct { + raw []byte + signature []byte +} + +func (m *certificateVerifyMsg) marshal() (x []byte) { + if m.raw != nil { + return m.raw + } + + // See http://tools.ietf.org/html/rfc4346#section-7.4.8 + siglength := len(m.signature) + length := 2 + siglength + x = make([]byte, 4+length) + x[0] = typeCertificateVerify + x[1] = uint8(length >> 16) + x[2] = uint8(length >> 8) + x[3] = uint8(length) + x[4] = uint8(siglength >> 8) + x[5] = uint8(siglength) + copy(x[6:], m.signature) + + m.raw = x + + return +} + +func (m *certificateVerifyMsg) unmarshal(data []byte) bool { + m.raw = data + + if len(data) < 6 { + return false + } + + length := uint32(data[1])<<16 | uint32(data[2])<<8 | uint32(data[3]) + if uint32(len(data))-4 != length { + return false + } + + siglength := int(data[4])<<8 + int(data[5]) + if len(data)-6 != siglength { + return false + } + + m.signature = data[6:] + + return true +} diff --git a/libgo/go/crypto/tls/handshake_messages_test.go b/libgo/go/crypto/tls/handshake_messages_test.go new file mode 100644 index 000000000..21577dd0b --- /dev/null +++ b/libgo/go/crypto/tls/handshake_messages_test.go @@ -0,0 +1,202 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tls + +import ( + "rand" + "reflect" + "testing" + "testing/quick" +) + +var tests = []interface{}{ + &clientHelloMsg{}, + &serverHelloMsg{}, + + &certificateMsg{}, + &certificateRequestMsg{}, + &certificateVerifyMsg{}, + &certificateStatusMsg{}, + &clientKeyExchangeMsg{}, + &finishedMsg{}, + &nextProtoMsg{}, +} + +type testMessage interface { + marshal() []byte + unmarshal([]byte) bool +} + +func TestMarshalUnmarshal(t *testing.T) { + rand := rand.New(rand.NewSource(0)) + for i, iface := range tests { + ty := reflect.NewValue(iface).Type() + + for j := 0; j < 100; j++ { + v, ok := quick.Value(ty, rand) + if !ok { + t.Errorf("#%d: failed to create value", i) + break + } + + m1 := v.Interface().(testMessage) + marshaled := m1.marshal() + m2 := iface.(testMessage) + if !m2.unmarshal(marshaled) { + t.Errorf("#%d failed to unmarshal %#v %x", i, m1, marshaled) + break + } + m2.marshal() // to fill any marshal cache in the message + + if !reflect.DeepEqual(m1, m2) { + t.Errorf("#%d got:%#v want:%#v %x", i, m2, m1, marshaled) + break + } + + if i >= 2 { + // The first two message types (ClientHello and + // ServerHello) are allowed to have parsable + // prefixes because the extension data is + // optional. + for j := 0; j < len(marshaled); j++ { + if m2.unmarshal(marshaled[0:j]) { + t.Errorf("#%d unmarshaled a prefix of length %d of %#v", i, j, m1) + break + } + } + } + } + } +} + +func TestFuzz(t *testing.T) { + rand := rand.New(rand.NewSource(0)) + for _, iface := range tests { + m := iface.(testMessage) + + for j := 0; j < 1000; j++ { + len := rand.Intn(100) + bytes := randomBytes(len, rand) + // This just looks for crashes due to bounds errors etc. + m.unmarshal(bytes) + } + } +} + +func randomBytes(n int, rand *rand.Rand) []byte { + r := make([]byte, n) + for i := 0; i < n; i++ { + r[i] = byte(rand.Int31()) + } + return r +} + +func randomString(n int, rand *rand.Rand) string { + b := randomBytes(n, rand) + return string(b) +} + +func (*clientHelloMsg) Generate(rand *rand.Rand, size int) reflect.Value { + m := &clientHelloMsg{} + m.vers = uint16(rand.Intn(65536)) + m.random = randomBytes(32, rand) + m.sessionId = randomBytes(rand.Intn(32), rand) + m.cipherSuites = make([]uint16, rand.Intn(63)+1) + for i := 0; i < len(m.cipherSuites); i++ { + m.cipherSuites[i] = uint16(rand.Int31()) + } + m.compressionMethods = randomBytes(rand.Intn(63)+1, rand) + if rand.Intn(10) > 5 { + m.nextProtoNeg = true + } + if rand.Intn(10) > 5 { + m.serverName = randomString(rand.Intn(255), rand) + } + m.ocspStapling = rand.Intn(10) > 5 + m.supportedPoints = randomBytes(rand.Intn(5)+1, rand) + m.supportedCurves = make([]uint16, rand.Intn(5)+1) + for i, _ := range m.supportedCurves { + m.supportedCurves[i] = uint16(rand.Intn(30000)) + } + + return reflect.NewValue(m) +} + +func (*serverHelloMsg) Generate(rand *rand.Rand, size int) reflect.Value { + m := &serverHelloMsg{} + m.vers = uint16(rand.Intn(65536)) + m.random = randomBytes(32, rand) + m.sessionId = randomBytes(rand.Intn(32), rand) + m.cipherSuite = uint16(rand.Int31()) + m.compressionMethod = uint8(rand.Intn(256)) + + if rand.Intn(10) > 5 { + m.nextProtoNeg = true + + n := rand.Intn(10) + m.nextProtos = make([]string, n) + for i := 0; i < n; i++ { + m.nextProtos[i] = randomString(20, rand) + } + } + + return reflect.NewValue(m) +} + +func (*certificateMsg) Generate(rand *rand.Rand, size int) reflect.Value { + m := &certificateMsg{} + numCerts := rand.Intn(20) + m.certificates = make([][]byte, numCerts) + for i := 0; i < numCerts; i++ { + m.certificates[i] = randomBytes(rand.Intn(10)+1, rand) + } + return reflect.NewValue(m) +} + +func (*certificateRequestMsg) Generate(rand *rand.Rand, size int) reflect.Value { + m := &certificateRequestMsg{} + m.certificateTypes = randomBytes(rand.Intn(5)+1, rand) + numCAs := rand.Intn(100) + m.certificateAuthorities = make([][]byte, numCAs) + for i := 0; i < numCAs; i++ { + m.certificateAuthorities[i] = randomBytes(rand.Intn(15)+1, rand) + } + return reflect.NewValue(m) +} + +func (*certificateVerifyMsg) Generate(rand *rand.Rand, size int) reflect.Value { + m := &certificateVerifyMsg{} + m.signature = randomBytes(rand.Intn(15)+1, rand) + return reflect.NewValue(m) +} + +func (*certificateStatusMsg) Generate(rand *rand.Rand, size int) reflect.Value { + m := &certificateStatusMsg{} + if rand.Intn(10) > 5 { + m.statusType = statusTypeOCSP + m.response = randomBytes(rand.Intn(10)+1, rand) + } else { + m.statusType = 42 + } + return reflect.NewValue(m) +} + +func (*clientKeyExchangeMsg) Generate(rand *rand.Rand, size int) reflect.Value { + m := &clientKeyExchangeMsg{} + m.ciphertext = randomBytes(rand.Intn(1000)+1, rand) + return reflect.NewValue(m) +} + +func (*finishedMsg) Generate(rand *rand.Rand, size int) reflect.Value { + m := &finishedMsg{} + m.verifyData = randomBytes(12, rand) + return reflect.NewValue(m) +} + +func (*nextProtoMsg) Generate(rand *rand.Rand, size int) reflect.Value { + m := &nextProtoMsg{} + m.proto = randomString(rand.Intn(255), rand) + return reflect.NewValue(m) +} diff --git a/libgo/go/crypto/tls/handshake_server.go b/libgo/go/crypto/tls/handshake_server.go new file mode 100644 index 000000000..955811ada --- /dev/null +++ b/libgo/go/crypto/tls/handshake_server.go @@ -0,0 +1,285 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tls + +import ( + "crypto/rsa" + "crypto/subtle" + "crypto/x509" + "io" + "os" +) + +func (c *Conn) serverHandshake() os.Error { + config := c.config + msg, err := c.readHandshake() + if err != nil { + return err + } + clientHello, ok := msg.(*clientHelloMsg) + if !ok { + return c.sendAlert(alertUnexpectedMessage) + } + vers, ok := mutualVersion(clientHello.vers) + if !ok { + return c.sendAlert(alertProtocolVersion) + } + c.vers = vers + c.haveVers = true + + finishedHash := newFinishedHash() + finishedHash.Write(clientHello.marshal()) + + hello := new(serverHelloMsg) + + supportedCurve := false +Curves: + for _, curve := range clientHello.supportedCurves { + switch curve { + case curveP256, curveP384, curveP521: + supportedCurve = true + break Curves + } + } + + supportedPointFormat := false + for _, pointFormat := range clientHello.supportedPoints { + if pointFormat == pointFormatUncompressed { + supportedPointFormat = true + break + } + } + + ellipticOk := supportedCurve && supportedPointFormat + + var suite *cipherSuite + var suiteId uint16 + for _, id := range clientHello.cipherSuites { + for _, supported := range config.cipherSuites() { + if id == supported { + suite = cipherSuites[id] + // Don't select a ciphersuite which we can't + // support for this client. + if suite.elliptic && !ellipticOk { + continue + } + suiteId = id + break + } + } + } + + foundCompression := false + // We only support null compression, so check that the client offered it. + for _, compression := range clientHello.compressionMethods { + if compression == compressionNone { + foundCompression = true + break + } + } + + if suite == nil || !foundCompression { + return c.sendAlert(alertHandshakeFailure) + } + + hello.vers = vers + hello.cipherSuite = suiteId + t := uint32(config.time()) + hello.random = make([]byte, 32) + hello.random[0] = byte(t >> 24) + hello.random[1] = byte(t >> 16) + hello.random[2] = byte(t >> 8) + hello.random[3] = byte(t) + _, err = io.ReadFull(config.rand(), hello.random[4:]) + if err != nil { + return c.sendAlert(alertInternalError) + } + hello.compressionMethod = compressionNone + if clientHello.nextProtoNeg { + hello.nextProtoNeg = true + hello.nextProtos = config.NextProtos + } + + finishedHash.Write(hello.marshal()) + c.writeRecord(recordTypeHandshake, hello.marshal()) + + if len(config.Certificates) == 0 { + return c.sendAlert(alertInternalError) + } + + certMsg := new(certificateMsg) + certMsg.certificates = config.Certificates[0].Certificate + finishedHash.Write(certMsg.marshal()) + c.writeRecord(recordTypeHandshake, certMsg.marshal()) + + keyAgreement := suite.ka() + + skx, err := keyAgreement.generateServerKeyExchange(config, clientHello, hello) + if err != nil { + c.sendAlert(alertHandshakeFailure) + return err + } + if skx != nil { + finishedHash.Write(skx.marshal()) + c.writeRecord(recordTypeHandshake, skx.marshal()) + } + + if config.AuthenticateClient { + // Request a client certificate + certReq := new(certificateRequestMsg) + certReq.certificateTypes = []byte{certTypeRSASign} + // An empty list of certificateAuthorities signals to + // the client that it may send any certificate in response + // to our request. + + finishedHash.Write(certReq.marshal()) + c.writeRecord(recordTypeHandshake, certReq.marshal()) + } + + helloDone := new(serverHelloDoneMsg) + finishedHash.Write(helloDone.marshal()) + c.writeRecord(recordTypeHandshake, helloDone.marshal()) + + var pub *rsa.PublicKey + if config.AuthenticateClient { + // Get client certificate + msg, err = c.readHandshake() + if err != nil { + return err + } + certMsg, ok = msg.(*certificateMsg) + if !ok { + return c.sendAlert(alertUnexpectedMessage) + } + finishedHash.Write(certMsg.marshal()) + + certs := make([]*x509.Certificate, len(certMsg.certificates)) + for i, asn1Data := range certMsg.certificates { + cert, err := x509.ParseCertificate(asn1Data) + if err != nil { + c.sendAlert(alertBadCertificate) + return os.ErrorString("could not parse client's certificate: " + err.String()) + } + certs[i] = cert + } + + // TODO(agl): do better validation of certs: max path length, name restrictions etc. + for i := 1; i < len(certs); i++ { + if err := certs[i-1].CheckSignatureFrom(certs[i]); err != nil { + c.sendAlert(alertBadCertificate) + return os.ErrorString("could not validate certificate signature: " + err.String()) + } + } + + if len(certs) > 0 { + key, ok := certs[0].PublicKey.(*rsa.PublicKey) + if !ok { + return c.sendAlert(alertUnsupportedCertificate) + } + pub = key + c.peerCertificates = certs + } + } + + // Get client key exchange + msg, err = c.readHandshake() + if err != nil { + return err + } + ckx, ok := msg.(*clientKeyExchangeMsg) + if !ok { + return c.sendAlert(alertUnexpectedMessage) + } + finishedHash.Write(ckx.marshal()) + + // If we received a client cert in response to our certificate request message, + // the client will send us a certificateVerifyMsg immediately after the + // clientKeyExchangeMsg. This message is a MD5SHA1 digest of all preceeding + // handshake-layer messages that is signed using the private key corresponding + // to the client's certificate. This allows us to verify that the client is in + // posession of the private key of the certificate. + if len(c.peerCertificates) > 0 { + msg, err = c.readHandshake() + if err != nil { + return err + } + certVerify, ok := msg.(*certificateVerifyMsg) + if !ok { + return c.sendAlert(alertUnexpectedMessage) + } + + digest := make([]byte, 36) + copy(digest[0:16], finishedHash.serverMD5.Sum()) + copy(digest[16:36], finishedHash.serverSHA1.Sum()) + err = rsa.VerifyPKCS1v15(pub, rsa.HashMD5SHA1, digest, certVerify.signature) + if err != nil { + c.sendAlert(alertBadCertificate) + return os.ErrorString("could not validate signature of connection nonces: " + err.String()) + } + + finishedHash.Write(certVerify.marshal()) + } + + preMasterSecret, err := keyAgreement.processClientKeyExchange(config, ckx) + if err != nil { + c.sendAlert(alertHandshakeFailure) + return err + } + + masterSecret, clientMAC, serverMAC, clientKey, serverKey, clientIV, serverIV := + keysFromPreMasterSecret10(preMasterSecret, clientHello.random, hello.random, suite.macLen, suite.keyLen, suite.ivLen) + + clientCipher := suite.cipher(clientKey, clientIV, true /* for reading */ ) + clientHash := suite.mac(clientMAC) + c.in.prepareCipherSpec(clientCipher, clientHash) + c.readRecord(recordTypeChangeCipherSpec) + if err := c.error(); err != nil { + return err + } + + if hello.nextProtoNeg { + msg, err = c.readHandshake() + if err != nil { + return err + } + nextProto, ok := msg.(*nextProtoMsg) + if !ok { + return c.sendAlert(alertUnexpectedMessage) + } + finishedHash.Write(nextProto.marshal()) + c.clientProtocol = nextProto.proto + } + + msg, err = c.readHandshake() + if err != nil { + return err + } + clientFinished, ok := msg.(*finishedMsg) + if !ok { + return c.sendAlert(alertUnexpectedMessage) + } + + verify := finishedHash.clientSum(masterSecret) + if len(verify) != len(clientFinished.verifyData) || + subtle.ConstantTimeCompare(verify, clientFinished.verifyData) != 1 { + return c.sendAlert(alertHandshakeFailure) + } + + finishedHash.Write(clientFinished.marshal()) + + serverCipher := suite.cipher(serverKey, serverIV, false /* not for reading */ ) + serverHash := suite.mac(serverMAC) + c.out.prepareCipherSpec(serverCipher, serverHash) + c.writeRecord(recordTypeChangeCipherSpec, []byte{1}) + + finished := new(finishedMsg) + finished.verifyData = finishedHash.serverSum(masterSecret) + c.writeRecord(recordTypeHandshake, finished.marshal()) + + c.handshakeComplete = true + c.cipherSuite = suiteId + + return nil +} diff --git a/libgo/go/crypto/tls/handshake_server_test.go b/libgo/go/crypto/tls/handshake_server_test.go new file mode 100644 index 000000000..5cf3ae049 --- /dev/null +++ b/libgo/go/crypto/tls/handshake_server_test.go @@ -0,0 +1,516 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tls + +import ( + "big" + "bytes" + "crypto/rsa" + "encoding/hex" + "flag" + "io" + "net" + "os" + "testing" +) + +type zeroSource struct{} + +func (zeroSource) Read(b []byte) (n int, err os.Error) { + for i := range b { + b[i] = 0 + } + + return len(b), nil +} + +var testConfig *Config + +func init() { + testConfig = new(Config) + testConfig.Time = func() int64 { return 0 } + testConfig.Rand = zeroSource{} + testConfig.Certificates = make([]Certificate, 1) + testConfig.Certificates[0].Certificate = [][]byte{testCertificate} + testConfig.Certificates[0].PrivateKey = testPrivateKey + testConfig.CipherSuites = []uint16{TLS_RSA_WITH_RC4_128_SHA} +} + +func testClientHelloFailure(t *testing.T, m handshakeMessage, expected os.Error) { + // Create in-memory network connection, + // send message to server. Should return + // expected error. + c, s := net.Pipe() + go func() { + cli := Client(c, testConfig) + if ch, ok := m.(*clientHelloMsg); ok { + cli.vers = ch.vers + } + cli.writeRecord(recordTypeHandshake, m.marshal()) + c.Close() + }() + err := Server(s, testConfig).Handshake() + s.Close() + if e, ok := err.(*net.OpError); !ok || e.Error != expected { + t.Errorf("Got error: %s; expected: %s", err, expected) + } +} + +func TestSimpleError(t *testing.T) { + testClientHelloFailure(t, &serverHelloDoneMsg{}, alertUnexpectedMessage) +} + +var badProtocolVersions = []uint16{0x0000, 0x0005, 0x0100, 0x0105, 0x0200, 0x0205, 0x0300} + +func TestRejectBadProtocolVersion(t *testing.T) { + for _, v := range badProtocolVersions { + testClientHelloFailure(t, &clientHelloMsg{vers: v}, alertProtocolVersion) + } +} + +func TestNoSuiteOverlap(t *testing.T) { + clientHello := &clientHelloMsg{nil, 0x0301, nil, nil, []uint16{0xff00}, []uint8{0}, false, "", false, nil, nil} + testClientHelloFailure(t, clientHello, alertHandshakeFailure) + +} + +func TestNoCompressionOverlap(t *testing.T) { + clientHello := &clientHelloMsg{nil, 0x0301, nil, nil, []uint16{TLS_RSA_WITH_RC4_128_SHA}, []uint8{0xff}, false, "", false, nil, nil} + testClientHelloFailure(t, clientHello, alertHandshakeFailure) +} + +func TestAlertForwarding(t *testing.T) { + c, s := net.Pipe() + go func() { + Client(c, testConfig).sendAlert(alertUnknownCA) + c.Close() + }() + + err := Server(s, testConfig).Handshake() + s.Close() + if e, ok := err.(*net.OpError); !ok || e.Error != os.Error(alertUnknownCA) { + t.Errorf("Got error: %s; expected: %s", err, alertUnknownCA) + } +} + +func TestClose(t *testing.T) { + c, s := net.Pipe() + go c.Close() + + err := Server(s, testConfig).Handshake() + s.Close() + if err != os.EOF { + t.Errorf("Got error: %s; expected: %s", err, os.EOF) + } +} + + +func testServerScript(t *testing.T, name string, serverScript [][]byte, config *Config) { + c, s := net.Pipe() + srv := Server(s, config) + go func() { + srv.Write([]byte("hello, world\n")) + srv.Close() + }() + + defer c.Close() + for i, b := range serverScript { + if i%2 == 0 { + c.Write(b) + continue + } + bb := make([]byte, len(b)) + _, err := io.ReadFull(c, bb) + if err != nil { + t.Fatalf("%s #%d: %s", name, i, err) + } + if !bytes.Equal(b, bb) { + t.Fatalf("%s #%d: mismatch on read: got:%x want:%x", name, i, bb, b) + } + } +} + +func TestHandshakeServerRC4(t *testing.T) { + testServerScript(t, "RC4", rc4ServerScript, testConfig) +} + +func TestHandshakeServerAES(t *testing.T) { + aesConfig := new(Config) + *aesConfig = *testConfig + aesConfig.CipherSuites = []uint16{TLS_RSA_WITH_AES_128_CBC_SHA} + testServerScript(t, "AES", aesServerScript, aesConfig) +} + +var serve = flag.Bool("serve", false, "run a TLS server on :10443") + +func TestRunServer(t *testing.T) { + if !*serve { + return + } + + l, err := Listen("tcp", ":10443", testConfig) + if err != nil { + t.Fatal(err) + } + + for { + c, err := l.Accept() + if err != nil { + break + } + _, err = c.Write([]byte("hello, world\n")) + if err != nil { + t.Errorf("error from TLS: %s", err) + break + } + c.Close() + } +} + +func bigFromString(s string) *big.Int { + ret := new(big.Int) + ret.SetString(s, 10) + return ret +} + +func fromHex(s string) []byte { + b, _ := hex.DecodeString(s) + return b +} + +var testCertificate = fromHex("308202b030820219a00302010202090085b0bba48a7fb8ca300d06092a864886f70d01010505003045310b3009060355040613024155311330110603550408130a536f6d652d53746174653121301f060355040a1318496e7465726e6574205769646769747320507479204c7464301e170d3130303432343039303933385a170d3131303432343039303933385a3045310b3009060355040613024155311330110603550408130a536f6d652d53746174653121301f060355040a1318496e7465726e6574205769646769747320507479204c746430819f300d06092a864886f70d010101050003818d0030818902818100bb79d6f517b5e5bf4610d0dc69bee62b07435ad0032d8a7a4385b71452e7a5654c2c78b8238cb5b482e5de1f953b7e62a52ca533d6fe125c7a56fcf506bffa587b263fb5cd04d3d0c921964ac7f4549f5abfef427100fe1899077f7e887d7df10439c4a22edb51c97ce3c04c3b326601cfafb11db8719a1ddbdb896baeda2d790203010001a381a73081a4301d0603551d0e04160414b1ade2855acfcb28db69ce2369ded3268e18883930750603551d23046e306c8014b1ade2855acfcb28db69ce2369ded3268e188839a149a4473045310b3009060355040613024155311330110603550408130a536f6d652d53746174653121301f060355040a1318496e7465726e6574205769646769747320507479204c746482090085b0bba48a7fb8ca300c0603551d13040530030101ff300d06092a864886f70d010105050003818100086c4524c76bb159ab0c52ccf2b014d7879d7a6475b55a9566e4c52b8eae12661feb4f38b36e60d392fdf74108b52513b1187a24fb301dbaed98b917ece7d73159db95d31d78ea50565cd5825a2d5a5f33c4b6d8c97590968c0f5298b5cd981f89205ff2a01ca31b9694dda9fd57e970e8266d71999b266e3850296c90a7bdd9") + +var testPrivateKey = &rsa.PrivateKey{ + PublicKey: rsa.PublicKey{ + N: bigFromString("131650079503776001033793877885499001334664249354723305978524647182322416328664556247316495448366990052837680518067798333412266673813370895702118944398081598789828837447552603077848001020611640547221687072142537202428102790818451901395596882588063427854225330436740647715202971973145151161964464812406232198521"), + E: 65537, + }, + D: bigFromString("29354450337804273969007277378287027274721892607543397931919078829901848876371746653677097639302788129485893852488285045793268732234230875671682624082413996177431586734171663258657462237320300610850244186316880055243099640544518318093544057213190320837094958164973959123058337475052510833916491060913053867729"), + P: bigFromString("11969277782311800166562047708379380720136961987713178380670422671426759650127150688426177829077494755200794297055316163155755835813760102405344560929062149"), + Q: bigFromString("10998999429884441391899182616418192492905073053684657075974935218461686523870125521822756579792315215543092255516093840728890783887287417039645833477273829"), +} + +// Script of interaction with gnutls implementation. +// The values for this test are obtained by building and running in server mode: +// % gotest -match "TestRunServer" -serve +// and then: +// % gnutls-cli --insecure --debug 100 -p 10443 localhost > /tmp/log 2>&1 +// % python parse-gnutls-cli-debug-log.py < /tmp/log +var rc4ServerScript = [][]byte{ + { + 0x16, 0x03, 0x02, 0x00, 0x7f, 0x01, 0x00, 0x00, + 0x7b, 0x03, 0x02, 0x4d, 0x08, 0x1f, 0x5a, 0x7a, + 0x0a, 0x92, 0x2f, 0xf0, 0x73, 0x16, 0x3a, 0x88, + 0x14, 0x85, 0x4c, 0x98, 0x15, 0x7b, 0x65, 0xe0, + 0x78, 0xd0, 0xed, 0xd0, 0xf3, 0x65, 0x20, 0xeb, + 0x80, 0xd1, 0x0b, 0x00, 0x00, 0x34, 0x00, 0x33, + 0x00, 0x45, 0x00, 0x39, 0x00, 0x88, 0x00, 0x16, + 0x00, 0x32, 0x00, 0x44, 0x00, 0x38, 0x00, 0x87, + 0x00, 0x13, 0x00, 0x66, 0x00, 0x90, 0x00, 0x91, + 0x00, 0x8f, 0x00, 0x8e, 0x00, 0x2f, 0x00, 0x41, + 0x00, 0x35, 0x00, 0x84, 0x00, 0x0a, 0x00, 0x05, + 0x00, 0x04, 0x00, 0x8c, 0x00, 0x8d, 0x00, 0x8b, + 0x00, 0x8a, 0x01, 0x00, 0x00, 0x1e, 0x00, 0x09, + 0x00, 0x03, 0x02, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x0e, 0x00, 0x0c, 0x00, 0x00, 0x09, 0x6c, 0x6f, + 0x63, 0x61, 0x6c, 0x68, 0x6f, 0x73, 0x74, 0xff, + 0x01, 0x00, 0x01, 0x00, + }, + + { + 0x16, 0x03, 0x01, 0x00, 0x2a, 0x02, 0x00, 0x00, + 0x26, 0x03, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x16, + 0x03, 0x01, 0x02, 0xbe, 0x0b, 0x00, 0x02, 0xba, + 0x00, 0x02, 0xb7, 0x00, 0x02, 0xb4, 0x30, 0x82, + 0x02, 0xb0, 0x30, 0x82, 0x02, 0x19, 0xa0, 0x03, + 0x02, 0x01, 0x02, 0x02, 0x09, 0x00, 0x85, 0xb0, + 0xbb, 0xa4, 0x8a, 0x7f, 0xb8, 0xca, 0x30, 0x0d, + 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, + 0x01, 0x01, 0x05, 0x05, 0x00, 0x30, 0x45, 0x31, + 0x0b, 0x30, 0x09, 0x06, 0x03, 0x55, 0x04, 0x06, + 0x13, 0x02, 0x41, 0x55, 0x31, 0x13, 0x30, 0x11, + 0x06, 0x03, 0x55, 0x04, 0x08, 0x13, 0x0a, 0x53, + 0x6f, 0x6d, 0x65, 0x2d, 0x53, 0x74, 0x61, 0x74, + 0x65, 0x31, 0x21, 0x30, 0x1f, 0x06, 0x03, 0x55, + 0x04, 0x0a, 0x13, 0x18, 0x49, 0x6e, 0x74, 0x65, + 0x72, 0x6e, 0x65, 0x74, 0x20, 0x57, 0x69, 0x64, + 0x67, 0x69, 0x74, 0x73, 0x20, 0x50, 0x74, 0x79, + 0x20, 0x4c, 0x74, 0x64, 0x30, 0x1e, 0x17, 0x0d, + 0x31, 0x30, 0x30, 0x34, 0x32, 0x34, 0x30, 0x39, + 0x30, 0x39, 0x33, 0x38, 0x5a, 0x17, 0x0d, 0x31, + 0x31, 0x30, 0x34, 0x32, 0x34, 0x30, 0x39, 0x30, + 0x39, 0x33, 0x38, 0x5a, 0x30, 0x45, 0x31, 0x0b, + 0x30, 0x09, 0x06, 0x03, 0x55, 0x04, 0x06, 0x13, + 0x02, 0x41, 0x55, 0x31, 0x13, 0x30, 0x11, 0x06, + 0x03, 0x55, 0x04, 0x08, 0x13, 0x0a, 0x53, 0x6f, + 0x6d, 0x65, 0x2d, 0x53, 0x74, 0x61, 0x74, 0x65, + 0x31, 0x21, 0x30, 0x1f, 0x06, 0x03, 0x55, 0x04, + 0x0a, 0x13, 0x18, 0x49, 0x6e, 0x74, 0x65, 0x72, + 0x6e, 0x65, 0x74, 0x20, 0x57, 0x69, 0x64, 0x67, + 0x69, 0x74, 0x73, 0x20, 0x50, 0x74, 0x79, 0x20, + 0x4c, 0x74, 0x64, 0x30, 0x81, 0x9f, 0x30, 0x0d, + 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, + 0x01, 0x01, 0x01, 0x05, 0x00, 0x03, 0x81, 0x8d, + 0x00, 0x30, 0x81, 0x89, 0x02, 0x81, 0x81, 0x00, + 0xbb, 0x79, 0xd6, 0xf5, 0x17, 0xb5, 0xe5, 0xbf, + 0x46, 0x10, 0xd0, 0xdc, 0x69, 0xbe, 0xe6, 0x2b, + 0x07, 0x43, 0x5a, 0xd0, 0x03, 0x2d, 0x8a, 0x7a, + 0x43, 0x85, 0xb7, 0x14, 0x52, 0xe7, 0xa5, 0x65, + 0x4c, 0x2c, 0x78, 0xb8, 0x23, 0x8c, 0xb5, 0xb4, + 0x82, 0xe5, 0xde, 0x1f, 0x95, 0x3b, 0x7e, 0x62, + 0xa5, 0x2c, 0xa5, 0x33, 0xd6, 0xfe, 0x12, 0x5c, + 0x7a, 0x56, 0xfc, 0xf5, 0x06, 0xbf, 0xfa, 0x58, + 0x7b, 0x26, 0x3f, 0xb5, 0xcd, 0x04, 0xd3, 0xd0, + 0xc9, 0x21, 0x96, 0x4a, 0xc7, 0xf4, 0x54, 0x9f, + 0x5a, 0xbf, 0xef, 0x42, 0x71, 0x00, 0xfe, 0x18, + 0x99, 0x07, 0x7f, 0x7e, 0x88, 0x7d, 0x7d, 0xf1, + 0x04, 0x39, 0xc4, 0xa2, 0x2e, 0xdb, 0x51, 0xc9, + 0x7c, 0xe3, 0xc0, 0x4c, 0x3b, 0x32, 0x66, 0x01, + 0xcf, 0xaf, 0xb1, 0x1d, 0xb8, 0x71, 0x9a, 0x1d, + 0xdb, 0xdb, 0x89, 0x6b, 0xae, 0xda, 0x2d, 0x79, + 0x02, 0x03, 0x01, 0x00, 0x01, 0xa3, 0x81, 0xa7, + 0x30, 0x81, 0xa4, 0x30, 0x1d, 0x06, 0x03, 0x55, + 0x1d, 0x0e, 0x04, 0x16, 0x04, 0x14, 0xb1, 0xad, + 0xe2, 0x85, 0x5a, 0xcf, 0xcb, 0x28, 0xdb, 0x69, + 0xce, 0x23, 0x69, 0xde, 0xd3, 0x26, 0x8e, 0x18, + 0x88, 0x39, 0x30, 0x75, 0x06, 0x03, 0x55, 0x1d, + 0x23, 0x04, 0x6e, 0x30, 0x6c, 0x80, 0x14, 0xb1, + 0xad, 0xe2, 0x85, 0x5a, 0xcf, 0xcb, 0x28, 0xdb, + 0x69, 0xce, 0x23, 0x69, 0xde, 0xd3, 0x26, 0x8e, + 0x18, 0x88, 0x39, 0xa1, 0x49, 0xa4, 0x47, 0x30, + 0x45, 0x31, 0x0b, 0x30, 0x09, 0x06, 0x03, 0x55, + 0x04, 0x06, 0x13, 0x02, 0x41, 0x55, 0x31, 0x13, + 0x30, 0x11, 0x06, 0x03, 0x55, 0x04, 0x08, 0x13, + 0x0a, 0x53, 0x6f, 0x6d, 0x65, 0x2d, 0x53, 0x74, + 0x61, 0x74, 0x65, 0x31, 0x21, 0x30, 0x1f, 0x06, + 0x03, 0x55, 0x04, 0x0a, 0x13, 0x18, 0x49, 0x6e, + 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x20, 0x57, + 0x69, 0x64, 0x67, 0x69, 0x74, 0x73, 0x20, 0x50, + 0x74, 0x79, 0x20, 0x4c, 0x74, 0x64, 0x82, 0x09, + 0x00, 0x85, 0xb0, 0xbb, 0xa4, 0x8a, 0x7f, 0xb8, + 0xca, 0x30, 0x0c, 0x06, 0x03, 0x55, 0x1d, 0x13, + 0x04, 0x05, 0x30, 0x03, 0x01, 0x01, 0xff, 0x30, + 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, + 0x0d, 0x01, 0x01, 0x05, 0x05, 0x00, 0x03, 0x81, + 0x81, 0x00, 0x08, 0x6c, 0x45, 0x24, 0xc7, 0x6b, + 0xb1, 0x59, 0xab, 0x0c, 0x52, 0xcc, 0xf2, 0xb0, + 0x14, 0xd7, 0x87, 0x9d, 0x7a, 0x64, 0x75, 0xb5, + 0x5a, 0x95, 0x66, 0xe4, 0xc5, 0x2b, 0x8e, 0xae, + 0x12, 0x66, 0x1f, 0xeb, 0x4f, 0x38, 0xb3, 0x6e, + 0x60, 0xd3, 0x92, 0xfd, 0xf7, 0x41, 0x08, 0xb5, + 0x25, 0x13, 0xb1, 0x18, 0x7a, 0x24, 0xfb, 0x30, + 0x1d, 0xba, 0xed, 0x98, 0xb9, 0x17, 0xec, 0xe7, + 0xd7, 0x31, 0x59, 0xdb, 0x95, 0xd3, 0x1d, 0x78, + 0xea, 0x50, 0x56, 0x5c, 0xd5, 0x82, 0x5a, 0x2d, + 0x5a, 0x5f, 0x33, 0xc4, 0xb6, 0xd8, 0xc9, 0x75, + 0x90, 0x96, 0x8c, 0x0f, 0x52, 0x98, 0xb5, 0xcd, + 0x98, 0x1f, 0x89, 0x20, 0x5f, 0xf2, 0xa0, 0x1c, + 0xa3, 0x1b, 0x96, 0x94, 0xdd, 0xa9, 0xfd, 0x57, + 0xe9, 0x70, 0xe8, 0x26, 0x6d, 0x71, 0x99, 0x9b, + 0x26, 0x6e, 0x38, 0x50, 0x29, 0x6c, 0x90, 0xa7, + 0xbd, 0xd9, 0x16, 0x03, 0x01, 0x00, 0x04, 0x0e, + 0x00, 0x00, 0x00, + }, + + { + 0x16, 0x03, 0x01, 0x00, 0x86, 0x10, 0x00, 0x00, + 0x82, 0x00, 0x80, 0x3c, 0x13, 0xd7, 0x12, 0xc1, + 0x6a, 0xf0, 0x3f, 0x8c, 0xa1, 0x35, 0x5d, 0xc5, + 0x89, 0x1e, 0x9e, 0xcd, 0x32, 0xc7, 0x9e, 0xe6, + 0xae, 0xd5, 0xf1, 0xbf, 0x70, 0xd7, 0xa9, 0xef, + 0x2c, 0x4c, 0xf4, 0x22, 0xbc, 0x17, 0x17, 0xaa, + 0x05, 0xf3, 0x9f, 0x80, 0xf2, 0xe9, 0x82, 0x2f, + 0x2a, 0x15, 0x54, 0x0d, 0x16, 0x0e, 0x77, 0x4c, + 0x28, 0x3c, 0x03, 0x2d, 0x2d, 0xd7, 0xc8, 0x64, + 0xd9, 0x59, 0x4b, 0x1c, 0xf4, 0xde, 0xff, 0x2f, + 0xbc, 0x94, 0xaf, 0x18, 0x26, 0x37, 0xce, 0x4f, + 0x84, 0x74, 0x2e, 0x45, 0x66, 0x7c, 0x0c, 0x54, + 0x46, 0x36, 0x5f, 0x65, 0x21, 0x7b, 0x83, 0x8c, + 0x6d, 0x76, 0xcd, 0x0d, 0x9f, 0xda, 0x1c, 0xa4, + 0x6e, 0xfe, 0xb1, 0xf7, 0x09, 0x0d, 0xfb, 0x74, + 0x66, 0x34, 0x99, 0x89, 0x7f, 0x5f, 0x77, 0x87, + 0x4a, 0x66, 0x4b, 0xa9, 0x59, 0x57, 0xe3, 0x56, + 0x0d, 0xdd, 0xd8, 0x14, 0x03, 0x01, 0x00, 0x01, + 0x01, 0x16, 0x03, 0x01, 0x00, 0x24, 0xc0, 0x4e, + 0xd3, 0x0f, 0xb5, 0xc0, 0x57, 0xa6, 0x18, 0x80, + 0x80, 0x6b, 0x49, 0xfe, 0xbd, 0x3a, 0x7a, 0x2c, + 0xef, 0x70, 0xb5, 0x1c, 0xd2, 0xdf, 0x5f, 0x78, + 0x5a, 0xd8, 0x4f, 0xa0, 0x95, 0xb4, 0xb3, 0xb5, + 0xaa, 0x3b, + }, + + { + 0x14, 0x03, 0x01, 0x00, 0x01, 0x01, 0x16, 0x03, + 0x01, 0x00, 0x24, 0x9d, 0xc9, 0xda, 0xdf, 0xeb, + 0xc8, 0xdb, 0xf8, 0x94, 0xa5, 0xef, 0xd5, 0xfc, + 0x89, 0x01, 0x64, 0x30, 0x77, 0x5a, 0x18, 0x4b, + 0x16, 0x79, 0x9c, 0xf6, 0xf5, 0x09, 0x22, 0x12, + 0x4c, 0x3e, 0xa8, 0x8e, 0x91, 0xa5, 0x24, + }, +} + +var aesServerScript = [][]byte{ + { + 0x16, 0x03, 0x02, 0x00, 0x7f, 0x01, 0x00, 0x00, + 0x7b, 0x03, 0x02, 0x4d, 0x08, 0x2d, 0x0b, 0xb3, + 0x57, 0x85, 0x71, 0x4b, 0xfb, 0x34, 0xab, 0x16, + 0xd4, 0x92, 0x50, 0x81, 0x16, 0x95, 0x11, 0x28, + 0x1a, 0xcb, 0xff, 0x09, 0x4d, 0x23, 0xa6, 0xfe, + 0x2e, 0xbb, 0x78, 0x00, 0x00, 0x34, 0x00, 0x33, + 0x00, 0x45, 0x00, 0x39, 0x00, 0x88, 0x00, 0x16, + 0x00, 0x32, 0x00, 0x44, 0x00, 0x38, 0x00, 0x87, + 0x00, 0x13, 0x00, 0x66, 0x00, 0x90, 0x00, 0x91, + 0x00, 0x8f, 0x00, 0x8e, 0x00, 0x2f, 0x00, 0x41, + 0x00, 0x35, 0x00, 0x84, 0x00, 0x0a, 0x00, 0x05, + 0x00, 0x04, 0x00, 0x8c, 0x00, 0x8d, 0x00, 0x8b, + 0x00, 0x8a, 0x01, 0x00, 0x00, 0x1e, 0x00, 0x09, + 0x00, 0x03, 0x02, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x0e, 0x00, 0x0c, 0x00, 0x00, 0x09, 0x6c, 0x6f, + 0x63, 0x61, 0x6c, 0x68, 0x6f, 0x73, 0x74, 0xff, + 0x01, 0x00, 0x01, 0x00, + }, + + { + 0x16, 0x03, 0x01, 0x00, 0x2a, 0x02, 0x00, 0x00, + 0x26, 0x03, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x2f, 0x00, 0x16, + 0x03, 0x01, 0x02, 0xbe, 0x0b, 0x00, 0x02, 0xba, + 0x00, 0x02, 0xb7, 0x00, 0x02, 0xb4, 0x30, 0x82, + 0x02, 0xb0, 0x30, 0x82, 0x02, 0x19, 0xa0, 0x03, + 0x02, 0x01, 0x02, 0x02, 0x09, 0x00, 0x85, 0xb0, + 0xbb, 0xa4, 0x8a, 0x7f, 0xb8, 0xca, 0x30, 0x0d, + 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, + 0x01, 0x01, 0x05, 0x05, 0x00, 0x30, 0x45, 0x31, + 0x0b, 0x30, 0x09, 0x06, 0x03, 0x55, 0x04, 0x06, + 0x13, 0x02, 0x41, 0x55, 0x31, 0x13, 0x30, 0x11, + 0x06, 0x03, 0x55, 0x04, 0x08, 0x13, 0x0a, 0x53, + 0x6f, 0x6d, 0x65, 0x2d, 0x53, 0x74, 0x61, 0x74, + 0x65, 0x31, 0x21, 0x30, 0x1f, 0x06, 0x03, 0x55, + 0x04, 0x0a, 0x13, 0x18, 0x49, 0x6e, 0x74, 0x65, + 0x72, 0x6e, 0x65, 0x74, 0x20, 0x57, 0x69, 0x64, + 0x67, 0x69, 0x74, 0x73, 0x20, 0x50, 0x74, 0x79, + 0x20, 0x4c, 0x74, 0x64, 0x30, 0x1e, 0x17, 0x0d, + 0x31, 0x30, 0x30, 0x34, 0x32, 0x34, 0x30, 0x39, + 0x30, 0x39, 0x33, 0x38, 0x5a, 0x17, 0x0d, 0x31, + 0x31, 0x30, 0x34, 0x32, 0x34, 0x30, 0x39, 0x30, + 0x39, 0x33, 0x38, 0x5a, 0x30, 0x45, 0x31, 0x0b, + 0x30, 0x09, 0x06, 0x03, 0x55, 0x04, 0x06, 0x13, + 0x02, 0x41, 0x55, 0x31, 0x13, 0x30, 0x11, 0x06, + 0x03, 0x55, 0x04, 0x08, 0x13, 0x0a, 0x53, 0x6f, + 0x6d, 0x65, 0x2d, 0x53, 0x74, 0x61, 0x74, 0x65, + 0x31, 0x21, 0x30, 0x1f, 0x06, 0x03, 0x55, 0x04, + 0x0a, 0x13, 0x18, 0x49, 0x6e, 0x74, 0x65, 0x72, + 0x6e, 0x65, 0x74, 0x20, 0x57, 0x69, 0x64, 0x67, + 0x69, 0x74, 0x73, 0x20, 0x50, 0x74, 0x79, 0x20, + 0x4c, 0x74, 0x64, 0x30, 0x81, 0x9f, 0x30, 0x0d, + 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, + 0x01, 0x01, 0x01, 0x05, 0x00, 0x03, 0x81, 0x8d, + 0x00, 0x30, 0x81, 0x89, 0x02, 0x81, 0x81, 0x00, + 0xbb, 0x79, 0xd6, 0xf5, 0x17, 0xb5, 0xe5, 0xbf, + 0x46, 0x10, 0xd0, 0xdc, 0x69, 0xbe, 0xe6, 0x2b, + 0x07, 0x43, 0x5a, 0xd0, 0x03, 0x2d, 0x8a, 0x7a, + 0x43, 0x85, 0xb7, 0x14, 0x52, 0xe7, 0xa5, 0x65, + 0x4c, 0x2c, 0x78, 0xb8, 0x23, 0x8c, 0xb5, 0xb4, + 0x82, 0xe5, 0xde, 0x1f, 0x95, 0x3b, 0x7e, 0x62, + 0xa5, 0x2c, 0xa5, 0x33, 0xd6, 0xfe, 0x12, 0x5c, + 0x7a, 0x56, 0xfc, 0xf5, 0x06, 0xbf, 0xfa, 0x58, + 0x7b, 0x26, 0x3f, 0xb5, 0xcd, 0x04, 0xd3, 0xd0, + 0xc9, 0x21, 0x96, 0x4a, 0xc7, 0xf4, 0x54, 0x9f, + 0x5a, 0xbf, 0xef, 0x42, 0x71, 0x00, 0xfe, 0x18, + 0x99, 0x07, 0x7f, 0x7e, 0x88, 0x7d, 0x7d, 0xf1, + 0x04, 0x39, 0xc4, 0xa2, 0x2e, 0xdb, 0x51, 0xc9, + 0x7c, 0xe3, 0xc0, 0x4c, 0x3b, 0x32, 0x66, 0x01, + 0xcf, 0xaf, 0xb1, 0x1d, 0xb8, 0x71, 0x9a, 0x1d, + 0xdb, 0xdb, 0x89, 0x6b, 0xae, 0xda, 0x2d, 0x79, + 0x02, 0x03, 0x01, 0x00, 0x01, 0xa3, 0x81, 0xa7, + 0x30, 0x81, 0xa4, 0x30, 0x1d, 0x06, 0x03, 0x55, + 0x1d, 0x0e, 0x04, 0x16, 0x04, 0x14, 0xb1, 0xad, + 0xe2, 0x85, 0x5a, 0xcf, 0xcb, 0x28, 0xdb, 0x69, + 0xce, 0x23, 0x69, 0xde, 0xd3, 0x26, 0x8e, 0x18, + 0x88, 0x39, 0x30, 0x75, 0x06, 0x03, 0x55, 0x1d, + 0x23, 0x04, 0x6e, 0x30, 0x6c, 0x80, 0x14, 0xb1, + 0xad, 0xe2, 0x85, 0x5a, 0xcf, 0xcb, 0x28, 0xdb, + 0x69, 0xce, 0x23, 0x69, 0xde, 0xd3, 0x26, 0x8e, + 0x18, 0x88, 0x39, 0xa1, 0x49, 0xa4, 0x47, 0x30, + 0x45, 0x31, 0x0b, 0x30, 0x09, 0x06, 0x03, 0x55, + 0x04, 0x06, 0x13, 0x02, 0x41, 0x55, 0x31, 0x13, + 0x30, 0x11, 0x06, 0x03, 0x55, 0x04, 0x08, 0x13, + 0x0a, 0x53, 0x6f, 0x6d, 0x65, 0x2d, 0x53, 0x74, + 0x61, 0x74, 0x65, 0x31, 0x21, 0x30, 0x1f, 0x06, + 0x03, 0x55, 0x04, 0x0a, 0x13, 0x18, 0x49, 0x6e, + 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x20, 0x57, + 0x69, 0x64, 0x67, 0x69, 0x74, 0x73, 0x20, 0x50, + 0x74, 0x79, 0x20, 0x4c, 0x74, 0x64, 0x82, 0x09, + 0x00, 0x85, 0xb0, 0xbb, 0xa4, 0x8a, 0x7f, 0xb8, + 0xca, 0x30, 0x0c, 0x06, 0x03, 0x55, 0x1d, 0x13, + 0x04, 0x05, 0x30, 0x03, 0x01, 0x01, 0xff, 0x30, + 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, + 0x0d, 0x01, 0x01, 0x05, 0x05, 0x00, 0x03, 0x81, + 0x81, 0x00, 0x08, 0x6c, 0x45, 0x24, 0xc7, 0x6b, + 0xb1, 0x59, 0xab, 0x0c, 0x52, 0xcc, 0xf2, 0xb0, + 0x14, 0xd7, 0x87, 0x9d, 0x7a, 0x64, 0x75, 0xb5, + 0x5a, 0x95, 0x66, 0xe4, 0xc5, 0x2b, 0x8e, 0xae, + 0x12, 0x66, 0x1f, 0xeb, 0x4f, 0x38, 0xb3, 0x6e, + 0x60, 0xd3, 0x92, 0xfd, 0xf7, 0x41, 0x08, 0xb5, + 0x25, 0x13, 0xb1, 0x18, 0x7a, 0x24, 0xfb, 0x30, + 0x1d, 0xba, 0xed, 0x98, 0xb9, 0x17, 0xec, 0xe7, + 0xd7, 0x31, 0x59, 0xdb, 0x95, 0xd3, 0x1d, 0x78, + 0xea, 0x50, 0x56, 0x5c, 0xd5, 0x82, 0x5a, 0x2d, + 0x5a, 0x5f, 0x33, 0xc4, 0xb6, 0xd8, 0xc9, 0x75, + 0x90, 0x96, 0x8c, 0x0f, 0x52, 0x98, 0xb5, 0xcd, + 0x98, 0x1f, 0x89, 0x20, 0x5f, 0xf2, 0xa0, 0x1c, + 0xa3, 0x1b, 0x96, 0x94, 0xdd, 0xa9, 0xfd, 0x57, + 0xe9, 0x70, 0xe8, 0x26, 0x6d, 0x71, 0x99, 0x9b, + 0x26, 0x6e, 0x38, 0x50, 0x29, 0x6c, 0x90, 0xa7, + 0xbd, 0xd9, 0x16, 0x03, 0x01, 0x00, 0x04, 0x0e, + 0x00, 0x00, 0x00, + }, + + { + 0x16, 0x03, 0x01, 0x00, 0x86, 0x10, 0x00, 0x00, + 0x82, 0x00, 0x80, 0x71, 0x9c, 0xe7, 0x23, 0xfc, + 0xb9, 0x19, 0x29, 0x82, 0xbf, 0xef, 0x08, 0xf7, + 0x99, 0x36, 0xc3, 0x4c, 0x6f, 0x05, 0xd2, 0x8b, + 0x62, 0x2b, 0x19, 0x9b, 0x7f, 0xc0, 0xcc, 0x48, + 0x30, 0x5f, 0xcd, 0xc3, 0x70, 0x55, 0x53, 0x73, + 0xfa, 0x79, 0x74, 0xf3, 0xa3, 0x76, 0x9f, 0xa1, + 0x7f, 0x98, 0xc2, 0xc0, 0xe3, 0xc5, 0xa0, 0x31, + 0x2f, 0xa6, 0xe8, 0x1e, 0x61, 0x46, 0xb3, 0x9b, + 0x4b, 0x16, 0xf1, 0x2d, 0xc7, 0x63, 0x7f, 0x79, + 0x22, 0x30, 0xd1, 0xf2, 0xfc, 0x77, 0x98, 0x0a, + 0x16, 0x11, 0x63, 0x71, 0x7f, 0x70, 0xef, 0x16, + 0xbb, 0x39, 0x87, 0x34, 0xac, 0x49, 0xbd, 0x07, + 0x67, 0xcb, 0x9c, 0xcc, 0xde, 0xef, 0xb1, 0xe0, + 0xdb, 0x01, 0xb5, 0x35, 0xa9, 0xb3, 0x10, 0x0c, + 0x4b, 0xee, 0xb3, 0x4e, 0xfd, 0xbe, 0x15, 0x27, + 0xf0, 0x46, 0xb2, 0x38, 0xba, 0x5f, 0xcc, 0x89, + 0xec, 0x29, 0x82, 0x14, 0x03, 0x01, 0x00, 0x01, + 0x01, 0x16, 0x03, 0x01, 0x00, 0x30, 0x3c, 0xfb, + 0xa4, 0x12, 0xcb, 0x00, 0xf9, 0x57, 0x7e, 0x9b, + 0xc9, 0xdc, 0x0c, 0xba, 0x9a, 0x81, 0x62, 0xfb, + 0x26, 0x13, 0x53, 0xfe, 0xaa, 0xcc, 0x82, 0xbb, + 0xb6, 0x67, 0x7f, 0x39, 0xbe, 0x4d, 0xbb, 0xc0, + 0x6c, 0x24, 0x31, 0x83, 0xa5, 0x50, 0x3a, 0x75, + 0x32, 0x64, 0xb5, 0xdb, 0xbe, 0x0a, + }, + + { + 0x14, 0x03, 0x01, 0x00, 0x01, 0x01, 0x16, 0x03, + 0x01, 0x00, 0x30, 0x43, 0x24, 0x42, 0x55, 0x08, + 0xe4, 0xc2, 0x15, 0xc9, 0xdb, 0x71, 0x69, 0xee, + 0x09, 0xc5, 0x1c, 0xfd, 0x46, 0x10, 0xa0, 0x68, + 0x21, 0xf2, 0x48, 0xac, 0x6c, 0xc0, 0x2b, 0x62, + 0x07, 0x8f, 0x48, 0x33, 0x0a, 0x6b, 0x62, 0x28, + 0x2e, 0x2c, 0xad, 0xcb, 0x34, 0x85, 0xca, 0x2e, + 0xcd, 0x84, 0xf0, + }, +} diff --git a/libgo/go/crypto/tls/key_agreement.go b/libgo/go/crypto/tls/key_agreement.go new file mode 100644 index 000000000..861c64f04 --- /dev/null +++ b/libgo/go/crypto/tls/key_agreement.go @@ -0,0 +1,246 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tls + +import ( + "big" + "crypto/elliptic" + "crypto/md5" + "crypto/rsa" + "crypto/sha1" + "crypto/x509" + "io" + "os" +) + +// rsaKeyAgreement implements the standard TLS key agreement where the client +// encrypts the pre-master secret to the server's public key. +type rsaKeyAgreement struct{} + +func (ka rsaKeyAgreement) generateServerKeyExchange(config *Config, clientHello *clientHelloMsg, hello *serverHelloMsg) (*serverKeyExchangeMsg, os.Error) { + return nil, nil +} + +func (ka rsaKeyAgreement) processClientKeyExchange(config *Config, ckx *clientKeyExchangeMsg) ([]byte, os.Error) { + preMasterSecret := make([]byte, 48) + _, err := io.ReadFull(config.rand(), preMasterSecret[2:]) + if err != nil { + return nil, err + } + + if len(ckx.ciphertext) < 2 { + return nil, os.ErrorString("bad ClientKeyExchange") + } + ciphertextLen := int(ckx.ciphertext[0])<<8 | int(ckx.ciphertext[1]) + if ciphertextLen != len(ckx.ciphertext)-2 { + return nil, os.ErrorString("bad ClientKeyExchange") + } + ciphertext := ckx.ciphertext[2:] + + err = rsa.DecryptPKCS1v15SessionKey(config.rand(), config.Certificates[0].PrivateKey, ciphertext, preMasterSecret) + if err != nil { + return nil, err + } + // We don't check the version number in the premaster secret. For one, + // by checking it, we would leak information about the validity of the + // encrypted pre-master secret. Secondly, it provides only a small + // benefit against a downgrade attack and some implementations send the + // wrong version anyway. See the discussion at the end of section + // 7.4.7.1 of RFC 4346. + return preMasterSecret, nil +} + +func (ka rsaKeyAgreement) processServerKeyExchange(config *Config, clientHello *clientHelloMsg, serverHello *serverHelloMsg, cert *x509.Certificate, skx *serverKeyExchangeMsg) os.Error { + return os.ErrorString("unexpected ServerKeyExchange") +} + +func (ka rsaKeyAgreement) generateClientKeyExchange(config *Config, clientHello *clientHelloMsg, cert *x509.Certificate) ([]byte, *clientKeyExchangeMsg, os.Error) { + preMasterSecret := make([]byte, 48) + preMasterSecret[0] = byte(clientHello.vers >> 8) + preMasterSecret[1] = byte(clientHello.vers) + _, err := io.ReadFull(config.rand(), preMasterSecret[2:]) + if err != nil { + return nil, nil, err + } + + encrypted, err := rsa.EncryptPKCS1v15(config.rand(), cert.PublicKey.(*rsa.PublicKey), preMasterSecret) + if err != nil { + return nil, nil, err + } + ckx := new(clientKeyExchangeMsg) + ckx.ciphertext = make([]byte, len(encrypted)+2) + ckx.ciphertext[0] = byte(len(encrypted) >> 8) + ckx.ciphertext[1] = byte(len(encrypted)) + copy(ckx.ciphertext[2:], encrypted) + return preMasterSecret, ckx, nil +} + + +// md5SHA1Hash implements TLS 1.0's hybrid hash function which consists of the +// concatenation of an MD5 and SHA1 hash. +func md5SHA1Hash(slices ...[]byte) []byte { + md5sha1 := make([]byte, md5.Size+sha1.Size) + hmd5 := md5.New() + for _, slice := range slices { + hmd5.Write(slice) + } + copy(md5sha1, hmd5.Sum()) + + hsha1 := sha1.New() + for _, slice := range slices { + hsha1.Write(slice) + } + copy(md5sha1[md5.Size:], hsha1.Sum()) + return md5sha1 +} + +// ecdheRSAKeyAgreement implements a TLS key agreement where the server +// generates a ephemeral EC public/private key pair and signs it. The +// pre-master secret is then calculated using ECDH. +type ecdheRSAKeyAgreement struct { + privateKey []byte + curve *elliptic.Curve + x, y *big.Int +} + +func (ka *ecdheRSAKeyAgreement) generateServerKeyExchange(config *Config, clientHello *clientHelloMsg, hello *serverHelloMsg) (*serverKeyExchangeMsg, os.Error) { + var curveid uint16 + +Curve: + for _, c := range clientHello.supportedCurves { + switch c { + case curveP256: + ka.curve = elliptic.P256() + curveid = c + break Curve + case curveP384: + ka.curve = elliptic.P384() + curveid = c + break Curve + case curveP521: + ka.curve = elliptic.P521() + curveid = c + break Curve + } + } + + var x, y *big.Int + var err os.Error + ka.privateKey, x, y, err = ka.curve.GenerateKey(config.rand()) + if err != nil { + return nil, err + } + ecdhePublic := ka.curve.Marshal(x, y) + + // http://tools.ietf.org/html/rfc4492#section-5.4 + serverECDHParams := make([]byte, 1+2+1+len(ecdhePublic)) + serverECDHParams[0] = 3 // named curve + serverECDHParams[1] = byte(curveid >> 8) + serverECDHParams[2] = byte(curveid) + serverECDHParams[3] = byte(len(ecdhePublic)) + copy(serverECDHParams[4:], ecdhePublic) + + md5sha1 := md5SHA1Hash(clientHello.random, hello.random, serverECDHParams) + sig, err := rsa.SignPKCS1v15(config.rand(), config.Certificates[0].PrivateKey, rsa.HashMD5SHA1, md5sha1) + if err != nil { + return nil, os.ErrorString("failed to sign ECDHE parameters: " + err.String()) + } + + skx := new(serverKeyExchangeMsg) + skx.key = make([]byte, len(serverECDHParams)+2+len(sig)) + copy(skx.key, serverECDHParams) + k := skx.key[len(serverECDHParams):] + k[0] = byte(len(sig) >> 8) + k[1] = byte(len(sig)) + copy(k[2:], sig) + + return skx, nil +} + +func (ka *ecdheRSAKeyAgreement) processClientKeyExchange(config *Config, ckx *clientKeyExchangeMsg) ([]byte, os.Error) { + if len(ckx.ciphertext) == 0 || int(ckx.ciphertext[0]) != len(ckx.ciphertext)-1 { + return nil, os.ErrorString("bad ClientKeyExchange") + } + x, y := ka.curve.Unmarshal(ckx.ciphertext[1:]) + if x == nil { + return nil, os.ErrorString("bad ClientKeyExchange") + } + x, _ = ka.curve.ScalarMult(x, y, ka.privateKey) + preMasterSecret := make([]byte, (ka.curve.BitSize+7)>>3) + xBytes := x.Bytes() + copy(preMasterSecret[len(preMasterSecret)-len(xBytes):], xBytes) + + return preMasterSecret, nil +} + +func (ka *ecdheRSAKeyAgreement) processServerKeyExchange(config *Config, clientHello *clientHelloMsg, serverHello *serverHelloMsg, cert *x509.Certificate, skx *serverKeyExchangeMsg) os.Error { + if len(skx.key) < 4 { + goto Error + } + if skx.key[0] != 3 { // named curve + return os.ErrorString("server selected unsupported curve") + } + curveid := uint16(skx.key[1])<<8 | uint16(skx.key[2]) + + switch curveid { + case curveP256: + ka.curve = elliptic.P256() + case curveP384: + ka.curve = elliptic.P384() + case curveP521: + ka.curve = elliptic.P521() + default: + return os.ErrorString("server selected unsupported curve") + } + + publicLen := int(skx.key[3]) + if publicLen+4 > len(skx.key) { + goto Error + } + ka.x, ka.y = ka.curve.Unmarshal(skx.key[4 : 4+publicLen]) + if ka.x == nil { + goto Error + } + serverECDHParams := skx.key[:4+publicLen] + + sig := skx.key[4+publicLen:] + if len(sig) < 2 { + goto Error + } + sigLen := int(sig[0])<<8 | int(sig[1]) + if sigLen+2 != len(sig) { + goto Error + } + sig = sig[2:] + + md5sha1 := md5SHA1Hash(clientHello.random, serverHello.random, serverECDHParams) + return rsa.VerifyPKCS1v15(cert.PublicKey.(*rsa.PublicKey), rsa.HashMD5SHA1, md5sha1, sig) + +Error: + return os.ErrorString("invalid ServerKeyExchange") +} + +func (ka *ecdheRSAKeyAgreement) generateClientKeyExchange(config *Config, clientHello *clientHelloMsg, cert *x509.Certificate) ([]byte, *clientKeyExchangeMsg, os.Error) { + if ka.curve == nil { + return nil, nil, os.ErrorString("missing ServerKeyExchange message") + } + priv, mx, my, err := ka.curve.GenerateKey(config.rand()) + if err != nil { + return nil, nil, err + } + x, _ := ka.curve.ScalarMult(ka.x, ka.y, priv) + preMasterSecret := make([]byte, (ka.curve.BitSize+7)>>3) + xBytes := x.Bytes() + copy(preMasterSecret[len(preMasterSecret)-len(xBytes):], xBytes) + + serialised := ka.curve.Marshal(mx, my) + + ckx := new(clientKeyExchangeMsg) + ckx.ciphertext = make([]byte, 1+len(serialised)) + ckx.ciphertext[0] = byte(len(serialised)) + copy(ckx.ciphertext[1:], serialised) + + return preMasterSecret, ckx, nil +} diff --git a/libgo/go/crypto/tls/prf.go b/libgo/go/crypto/tls/prf.go new file mode 100644 index 000000000..478cf65f9 --- /dev/null +++ b/libgo/go/crypto/tls/prf.go @@ -0,0 +1,153 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tls + +import ( + "crypto/hmac" + "crypto/md5" + "crypto/sha1" + "hash" + "os" +) + +// Split a premaster secret in two as specified in RFC 4346, section 5. +func splitPreMasterSecret(secret []byte) (s1, s2 []byte) { + s1 = secret[0 : (len(secret)+1)/2] + s2 = secret[len(secret)/2:] + return +} + +// pHash implements the P_hash function, as defined in RFC 4346, section 5. +func pHash(result, secret, seed []byte, hash func() hash.Hash) { + h := hmac.New(hash, secret) + h.Write(seed) + a := h.Sum() + + j := 0 + for j < len(result) { + h.Reset() + h.Write(a) + h.Write(seed) + b := h.Sum() + todo := len(b) + if j+todo > len(result) { + todo = len(result) - j + } + copy(result[j:j+todo], b) + j += todo + + h.Reset() + h.Write(a) + a = h.Sum() + } +} + +// pRF10 implements the TLS 1.0 pseudo-random function, as defined in RFC 2246, section 5. +func pRF10(result, secret, label, seed []byte) { + hashSHA1 := sha1.New + hashMD5 := md5.New + + labelAndSeed := make([]byte, len(label)+len(seed)) + copy(labelAndSeed, label) + copy(labelAndSeed[len(label):], seed) + + s1, s2 := splitPreMasterSecret(secret) + pHash(result, s1, labelAndSeed, hashMD5) + result2 := make([]byte, len(result)) + pHash(result2, s2, labelAndSeed, hashSHA1) + + for i, b := range result2 { + result[i] ^= b + } +} + +const ( + tlsRandomLength = 32 // Length of a random nonce in TLS 1.1. + masterSecretLength = 48 // Length of a master secret in TLS 1.1. + finishedVerifyLength = 12 // Length of verify_data in a Finished message. +) + +var masterSecretLabel = []byte("master secret") +var keyExpansionLabel = []byte("key expansion") +var clientFinishedLabel = []byte("client finished") +var serverFinishedLabel = []byte("server finished") + +// keysFromPreMasterSecret generates the connection keys from the pre master +// secret, given the lengths of the MAC key, cipher key and IV, as defined in +// RFC 2246, section 6.3. +func keysFromPreMasterSecret10(preMasterSecret, clientRandom, serverRandom []byte, macLen, keyLen, ivLen int) (masterSecret, clientMAC, serverMAC, clientKey, serverKey, clientIV, serverIV []byte) { + var seed [tlsRandomLength * 2]byte + copy(seed[0:len(clientRandom)], clientRandom) + copy(seed[len(clientRandom):], serverRandom) + masterSecret = make([]byte, masterSecretLength) + pRF10(masterSecret, preMasterSecret, masterSecretLabel, seed[0:]) + + copy(seed[0:len(clientRandom)], serverRandom) + copy(seed[len(serverRandom):], clientRandom) + + n := 2*macLen + 2*keyLen + 2*ivLen + keyMaterial := make([]byte, n) + pRF10(keyMaterial, masterSecret, keyExpansionLabel, seed[0:]) + clientMAC = keyMaterial[:macLen] + keyMaterial = keyMaterial[macLen:] + serverMAC = keyMaterial[:macLen] + keyMaterial = keyMaterial[macLen:] + clientKey = keyMaterial[:keyLen] + keyMaterial = keyMaterial[keyLen:] + serverKey = keyMaterial[:keyLen] + keyMaterial = keyMaterial[keyLen:] + clientIV = keyMaterial[:ivLen] + keyMaterial = keyMaterial[ivLen:] + serverIV = keyMaterial[:ivLen] + return +} + +// A finishedHash calculates the hash of a set of handshake messages suitable +// for including in a Finished message. +type finishedHash struct { + clientMD5 hash.Hash + clientSHA1 hash.Hash + serverMD5 hash.Hash + serverSHA1 hash.Hash +} + +func newFinishedHash() finishedHash { + return finishedHash{md5.New(), sha1.New(), md5.New(), sha1.New()} +} + +func (h finishedHash) Write(msg []byte) (n int, err os.Error) { + h.clientMD5.Write(msg) + h.clientSHA1.Write(msg) + h.serverMD5.Write(msg) + h.serverSHA1.Write(msg) + return len(msg), nil +} + +// finishedSum calculates the contents of the verify_data member of a Finished +// message given the MD5 and SHA1 hashes of a set of handshake messages. +func finishedSum(md5, sha1, label, masterSecret []byte) []byte { + seed := make([]byte, len(md5)+len(sha1)) + copy(seed, md5) + copy(seed[len(md5):], sha1) + out := make([]byte, finishedVerifyLength) + pRF10(out, masterSecret, label, seed) + return out +} + +// clientSum returns the contents of the verify_data member of a client's +// Finished message. +func (h finishedHash) clientSum(masterSecret []byte) []byte { + md5 := h.clientMD5.Sum() + sha1 := h.clientSHA1.Sum() + return finishedSum(md5, sha1, clientFinishedLabel, masterSecret) +} + +// serverSum returns the contents of the verify_data member of a server's +// Finished message. +func (h finishedHash) serverSum(masterSecret []byte) []byte { + md5 := h.serverMD5.Sum() + sha1 := h.serverSHA1.Sum() + return finishedSum(md5, sha1, serverFinishedLabel, masterSecret) +} diff --git a/libgo/go/crypto/tls/prf_test.go b/libgo/go/crypto/tls/prf_test.go new file mode 100644 index 000000000..f8c4acb9d --- /dev/null +++ b/libgo/go/crypto/tls/prf_test.go @@ -0,0 +1,104 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tls + +import ( + "encoding/hex" + "testing" +) + +type testSplitPreMasterSecretTest struct { + in, out1, out2 string +} + +var testSplitPreMasterSecretTests = []testSplitPreMasterSecretTest{ + {"", "", ""}, + {"00", "00", "00"}, + {"0011", "00", "11"}, + {"001122", "0011", "1122"}, + {"00112233", "0011", "2233"}, +} + +func TestSplitPreMasterSecret(t *testing.T) { + for i, test := range testSplitPreMasterSecretTests { + in, _ := hex.DecodeString(test.in) + out1, out2 := splitPreMasterSecret(in) + s1 := hex.EncodeToString(out1) + s2 := hex.EncodeToString(out2) + if s1 != test.out1 || s2 != test.out2 { + t.Errorf("#%d: got: (%s, %s) want: (%s, %s)", i, s1, s2, test.out1, test.out2) + } + } +} + +type testKeysFromTest struct { + preMasterSecret string + clientRandom, serverRandom string + masterSecret string + clientMAC, serverMAC string + clientKey, serverKey string + macLen, keyLen int +} + +func TestKeysFromPreMasterSecret(t *testing.T) { + for i, test := range testKeysFromTests { + in, _ := hex.DecodeString(test.preMasterSecret) + clientRandom, _ := hex.DecodeString(test.clientRandom) + serverRandom, _ := hex.DecodeString(test.serverRandom) + master, clientMAC, serverMAC, clientKey, serverKey, _, _ := keysFromPreMasterSecret10(in, clientRandom, serverRandom, test.macLen, test.keyLen, 0) + masterString := hex.EncodeToString(master) + clientMACString := hex.EncodeToString(clientMAC) + serverMACString := hex.EncodeToString(serverMAC) + clientKeyString := hex.EncodeToString(clientKey) + serverKeyString := hex.EncodeToString(serverKey) + if masterString != test.masterSecret || + clientMACString != test.clientMAC || + serverMACString != test.serverMAC || + clientKeyString != test.clientKey || + serverKeyString != test.serverKey { + t.Errorf("#%d: got: (%s, %s, %s, %s, %s) want: (%s, %s, %s, %s %s)", i, masterString, clientMACString, serverMACString, clientKeyString, serverMACString, test.masterSecret, test.clientMAC, test.serverMAC, test.clientKey, test.serverKey) + } + } +} + +// These test vectors were generated from GnuTLS using `gnutls-cli --insecure -d 9 ` +var testKeysFromTests = []testKeysFromTest{ + { + "0302cac83ad4b1db3b9ab49ad05957de2a504a634a386fc600889321e1a971f57479466830ac3e6f468e87f5385fa0c5", + "4ae66303755184a3917fcb44880605fcc53baa01912b22ed94473fc69cebd558", + "4ae663020ec16e6bb5130be918cfcafd4d765979a3136a5d50c593446e4e44db", + "3d851bab6e5556e959a16bc36d66cfae32f672bfa9ecdef6096cbb1b23472df1da63dbbd9827606413221d149ed08ceb", + "805aaa19b3d2c0a0759a4b6c9959890e08480119", + "2d22f9fe519c075c16448305ceee209fc24ad109", + "d50b5771244f850cd8117a9ccafe2cf1", + "e076e33206b30507a85c32855acd0919", + 20, + 16, + }, + { + "03023f7527316bc12cbcd69e4b9e8275d62c028f27e65c745cfcddc7ce01bd3570a111378b63848127f1c36e5f9e4890", + "4ae66364b5ea56b20ce4e25555aed2d7e67f42788dd03f3fee4adae0459ab106", + "4ae66363ab815cbf6a248b87d6b556184e945e9b97fbdf247858b0bdafacfa1c", + "7d64be7c80c59b740200b4b9c26d0baaa1c5ae56705acbcf2307fe62beb4728c19392c83f20483801cce022c77645460", + "97742ed60a0554ca13f04f97ee193177b971e3b0", + "37068751700400e03a8477a5c7eec0813ab9e0dc", + "207cddbc600d2a200abac6502053ee5c", + "df3f94f6e1eacc753b815fe16055cd43", + 20, + 16, + }, + { + "832d515f1d61eebb2be56ba0ef79879efb9b527504abb386fb4310ed5d0e3b1f220d3bb6b455033a2773e6d8bdf951d278a187482b400d45deb88a5d5a6bb7d6a7a1decc04eb9ef0642876cd4a82d374d3b6ff35f0351dc5d411104de431375355addc39bfb1f6329fb163b0bc298d658338930d07d313cd980a7e3d9196cac1", + "4ae663b2ee389c0de147c509d8f18f5052afc4aaf9699efe8cb05ece883d3a5e", + "4ae664d503fd4cff50cfc1fb8fc606580f87b0fcdac9554ba0e01d785bdf278e", + "1aff2e7a2c4279d0126f57a65a77a8d9d0087cf2733366699bec27eb53d5740705a8574bb1acc2abbe90e44f0dd28d6c", + "3c7647c93c1379a31a609542aa44e7f117a70085", + "0d73102994be74a575a3ead8532590ca32a526d4", + "ac7581b0b6c10d85bbd905ffbf36c65e", + "ff07edde49682b45466bd2e39464b306", + 20, + 16, + }, +} diff --git a/libgo/go/crypto/tls/tls.go b/libgo/go/crypto/tls/tls.go new file mode 100644 index 000000000..b11d3225d --- /dev/null +++ b/libgo/go/crypto/tls/tls.go @@ -0,0 +1,167 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This package partially implements the TLS 1.1 protocol, as specified in RFC 4346. +package tls + +import ( + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "io/ioutil" + "net" + "os" + "strings" +) + +// Server returns a new TLS server side connection +// using conn as the underlying transport. +// The configuration config must be non-nil and must have +// at least one certificate. +func Server(conn net.Conn, config *Config) *Conn { + return &Conn{conn: conn, config: config} +} + +// Client returns a new TLS client side connection +// using conn as the underlying transport. +// Client interprets a nil configuration as equivalent to +// the zero configuration; see the documentation of Config +// for the defaults. +func Client(conn net.Conn, config *Config) *Conn { + return &Conn{conn: conn, config: config, isClient: true} +} + +// A Listener implements a network listener (net.Listener) for TLS connections. +type Listener struct { + listener net.Listener + config *Config +} + +// Accept waits for and returns the next incoming TLS connection. +// The returned connection c is a *tls.Conn. +func (l *Listener) Accept() (c net.Conn, err os.Error) { + c, err = l.listener.Accept() + if err != nil { + return + } + c = Server(c, l.config) + return +} + +// Close closes the listener. +func (l *Listener) Close() os.Error { return l.listener.Close() } + +// Addr returns the listener's network address. +func (l *Listener) Addr() net.Addr { return l.listener.Addr() } + +// NewListener creates a Listener which accepts connections from an inner +// Listener and wraps each connection with Server. +// The configuration config must be non-nil and must have +// at least one certificate. +func NewListener(listener net.Listener, config *Config) (l *Listener) { + l = new(Listener) + l.listener = listener + l.config = config + return +} + +// Listen creates a TLS listener accepting connections on the +// given network address using net.Listen. +// The configuration config must be non-nil and must have +// at least one certificate. +func Listen(network, laddr string, config *Config) (*Listener, os.Error) { + if config == nil || len(config.Certificates) == 0 { + return nil, os.NewError("tls.Listen: no certificates in configuration") + } + l, err := net.Listen(network, laddr) + if err != nil { + return nil, err + } + return NewListener(l, config), nil +} + +// Dial connects to the given network address using net.Dial +// and then initiates a TLS handshake, returning the resulting +// TLS connection. +// Dial interprets a nil configuration as equivalent to +// the zero configuration; see the documentation of Config +// for the defaults. +func Dial(network, laddr, raddr string, config *Config) (*Conn, os.Error) { + c, err := net.Dial(network, laddr, raddr) + if err != nil { + return nil, err + } + + colonPos := strings.LastIndex(raddr, ":") + if colonPos == -1 { + colonPos = len(raddr) + } + hostname := raddr[:colonPos] + + if config == nil { + config = defaultConfig() + } + if config.ServerName != "" { + // Make a copy to avoid polluting argument or default. + c := *config + c.ServerName = hostname + config = &c + } + conn := Client(c, config) + if err = conn.Handshake(); err != nil { + c.Close() + return nil, err + } + return conn, nil +} + +// LoadX509KeyPair reads and parses a public/private key pair from a pair of +// files. The files must contain PEM encoded data. +func LoadX509KeyPair(certFile string, keyFile string) (cert Certificate, err os.Error) { + certPEMBlock, err := ioutil.ReadFile(certFile) + if err != nil { + return + } + + certDERBlock, _ := pem.Decode(certPEMBlock) + if certDERBlock == nil { + err = os.ErrorString("crypto/tls: failed to parse certificate PEM data") + return + } + + cert.Certificate = [][]byte{certDERBlock.Bytes} + + keyPEMBlock, err := ioutil.ReadFile(keyFile) + if err != nil { + return + } + + keyDERBlock, _ := pem.Decode(keyPEMBlock) + if keyDERBlock == nil { + err = os.ErrorString("crypto/tls: failed to parse key PEM data") + return + } + + key, err := x509.ParsePKCS1PrivateKey(keyDERBlock.Bytes) + if err != nil { + err = os.ErrorString("crypto/tls: failed to parse key") + return + } + + cert.PrivateKey = key + + // We don't need to parse the public key for TLS, but we so do anyway + // to check that it looks sane and matches the private key. + x509Cert, err := x509.ParseCertificate(certDERBlock.Bytes) + if err != nil { + return + } + + if x509Cert.PublicKeyAlgorithm != x509.RSA || x509Cert.PublicKey.(*rsa.PublicKey).N.Cmp(key.PublicKey.N) != 0 { + err = os.ErrorString("crypto/tls: private key does not match public key") + return + } + + return +} diff --git a/libgo/go/crypto/twofish/twofish.go b/libgo/go/crypto/twofish/twofish.go new file mode 100644 index 000000000..62253e797 --- /dev/null +++ b/libgo/go/crypto/twofish/twofish.go @@ -0,0 +1,358 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This package implements Bruce Schneier's Twofish encryption algorithm. +package twofish + +// Twofish is defined in http://www.schneier.com/paper-twofish-paper.pdf [TWOFISH] + +// This code is a port of the LibTom C implementation. +// See http://libtom.org/?page=features&newsitems=5&whatfile=crypt. +// LibTomCrypt is free for all purposes under the public domain. +// It was heavily inspired by the go blowfish package. + +import ( + "os" + "strconv" +) + +// BlockSize is the constant block size of Twofish. +const BlockSize = 16 + +const mdsPolynomial = 0x169 // x^8 + x^6 + x^5 + x^3 + 1, see [TWOFISH] 4.2 +const rsPolynomial = 0x14d // x^8 + x^6 + x^3 + x^2 + 1, see [TWOFISH] 4.3 + +// A Cipher is an instance of Twofish encryption using a particular key. +type Cipher struct { + s [4][256]uint32 + k [40]uint32 +} + +type KeySizeError int + +func (k KeySizeError) String() string { + return "crypto/twofish: invalid key size " + strconv.Itoa(int(k)) +} + +// NewCipher creates and returns a Cipher. +// The key argument should be the Twofish key, 16, 24 or 32 bytes. +func NewCipher(key []byte) (*Cipher, os.Error) { + keylen := len(key) + + if keylen != 16 && keylen != 24 && keylen != 32 { + return nil, KeySizeError(keylen) + } + + // k is the number of 64 bit words in key + k := keylen / 8 + + // Create the S[..] words + var S [4 * 4]byte + for i := 0; i < k; i++ { + // Computes [y0 y1 y2 y3] = rs . [x0 x1 x2 x3 x4 x5 x6 x7] + for j, rsRow := range rs { + for k, rsVal := range rsRow { + S[4*i+j] ^= gfMult(key[8*i+k], rsVal, rsPolynomial) + } + } + } + + // Calculate subkeys + c := new(Cipher) + var tmp [4]byte + for i := byte(0); i < 20; i++ { + // A = h(p * 2x, Me) + for j := range tmp { + tmp[j] = 2 * i + } + A := h(tmp[:], key, 0) + + // B = rolc(h(p * (2x + 1), Mo), 8) + for j := range tmp { + tmp[j] = 2*i + 1 + } + B := h(tmp[:], key, 1) + B = rol(B, 8) + + c.k[2*i] = A + B + + // K[2i+1] = (A + 2B) <<< 9 + c.k[2*i+1] = rol(2*B+A, 9) + } + + // Calculate sboxes + switch k { + case 2: + for i := range c.s[0] { + c.s[0][i] = mdsColumnMult(sbox[1][sbox[0][sbox[0][byte(i)]^S[0]]^S[4]], 0) + c.s[1][i] = mdsColumnMult(sbox[0][sbox[0][sbox[1][byte(i)]^S[1]]^S[5]], 1) + c.s[2][i] = mdsColumnMult(sbox[1][sbox[1][sbox[0][byte(i)]^S[2]]^S[6]], 2) + c.s[3][i] = mdsColumnMult(sbox[0][sbox[1][sbox[1][byte(i)]^S[3]]^S[7]], 3) + } + case 3: + for i := range c.s[0] { + c.s[0][i] = mdsColumnMult(sbox[1][sbox[0][sbox[0][sbox[1][byte(i)]^S[0]]^S[4]]^S[8]], 0) + c.s[1][i] = mdsColumnMult(sbox[0][sbox[0][sbox[1][sbox[1][byte(i)]^S[1]]^S[5]]^S[9]], 1) + c.s[2][i] = mdsColumnMult(sbox[1][sbox[1][sbox[0][sbox[0][byte(i)]^S[2]]^S[6]]^S[10]], 2) + c.s[3][i] = mdsColumnMult(sbox[0][sbox[1][sbox[1][sbox[0][byte(i)]^S[3]]^S[7]]^S[11]], 3) + } + default: + for i := range c.s[0] { + c.s[0][i] = mdsColumnMult(sbox[1][sbox[0][sbox[0][sbox[1][sbox[1][byte(i)]^S[0]]^S[4]]^S[8]]^S[12]], 0) + c.s[1][i] = mdsColumnMult(sbox[0][sbox[0][sbox[1][sbox[1][sbox[0][byte(i)]^S[1]]^S[5]]^S[9]]^S[13]], 1) + c.s[2][i] = mdsColumnMult(sbox[1][sbox[1][sbox[0][sbox[0][sbox[0][byte(i)]^S[2]]^S[6]]^S[10]]^S[14]], 2) + c.s[3][i] = mdsColumnMult(sbox[0][sbox[1][sbox[1][sbox[0][sbox[1][byte(i)]^S[3]]^S[7]]^S[11]]^S[15]], 3) + } + } + + return c, nil +} + +// Reset zeros the key data, so that it will no longer appear in the process's +// memory. +func (c *Cipher) Reset() { + for i := range c.k { + c.k[i] = 0 + } + for i := range c.s { + for j := 0; j < 265; j++ { + c.s[i][j] = 0 + } + } +} + +// BlockSize returns the Twofish block size, 16 bytes. +func (c *Cipher) BlockSize() int { return BlockSize } + +// store32l stores src in dst in little-endian form. +func store32l(dst []byte, src uint32) { + dst[0] = byte(src) + dst[1] = byte(src >> 8) + dst[2] = byte(src >> 16) + dst[3] = byte(src >> 24) + return +} + +// load32l reads a little-endian uint32 from src. +func load32l(src []byte) uint32 { + return uint32(src[0]) | uint32(src[1])<<8 | uint32(src[2])<<16 | uint32(src[3])<<24 +} + +// rol returns x after a left circular rotation of y bits. +func rol(x, y uint32) uint32 { + return (x << (y & 31)) | (x >> (32 - (y & 31))) +} + +// ror returns x after a right circular rotation of y bits. +func ror(x, y uint32) uint32 { + return (x >> (y & 31)) | (x << (32 - (y & 31))) +} + +// The RS matrix. See [TWOFISH] 4.3 +var rs = [4][8]byte{ + {0x01, 0xA4, 0x55, 0x87, 0x5A, 0x58, 0xDB, 0x9E}, + {0xA4, 0x56, 0x82, 0xF3, 0x1E, 0xC6, 0x68, 0xE5}, + {0x02, 0xA1, 0xFC, 0xC1, 0x47, 0xAE, 0x3D, 0x19}, + {0xA4, 0x55, 0x87, 0x5A, 0x58, 0xDB, 0x9E, 0x03}, +} + +// sbox tables +var sbox = [2][256]byte{ + { + 0xa9, 0x67, 0xb3, 0xe8, 0x04, 0xfd, 0xa3, 0x76, 0x9a, 0x92, 0x80, 0x78, 0xe4, 0xdd, 0xd1, 0x38, + 0x0d, 0xc6, 0x35, 0x98, 0x18, 0xf7, 0xec, 0x6c, 0x43, 0x75, 0x37, 0x26, 0xfa, 0x13, 0x94, 0x48, + 0xf2, 0xd0, 0x8b, 0x30, 0x84, 0x54, 0xdf, 0x23, 0x19, 0x5b, 0x3d, 0x59, 0xf3, 0xae, 0xa2, 0x82, + 0x63, 0x01, 0x83, 0x2e, 0xd9, 0x51, 0x9b, 0x7c, 0xa6, 0xeb, 0xa5, 0xbe, 0x16, 0x0c, 0xe3, 0x61, + 0xc0, 0x8c, 0x3a, 0xf5, 0x73, 0x2c, 0x25, 0x0b, 0xbb, 0x4e, 0x89, 0x6b, 0x53, 0x6a, 0xb4, 0xf1, + 0xe1, 0xe6, 0xbd, 0x45, 0xe2, 0xf4, 0xb6, 0x66, 0xcc, 0x95, 0x03, 0x56, 0xd4, 0x1c, 0x1e, 0xd7, + 0xfb, 0xc3, 0x8e, 0xb5, 0xe9, 0xcf, 0xbf, 0xba, 0xea, 0x77, 0x39, 0xaf, 0x33, 0xc9, 0x62, 0x71, + 0x81, 0x79, 0x09, 0xad, 0x24, 0xcd, 0xf9, 0xd8, 0xe5, 0xc5, 0xb9, 0x4d, 0x44, 0x08, 0x86, 0xe7, + 0xa1, 0x1d, 0xaa, 0xed, 0x06, 0x70, 0xb2, 0xd2, 0x41, 0x7b, 0xa0, 0x11, 0x31, 0xc2, 0x27, 0x90, + 0x20, 0xf6, 0x60, 0xff, 0x96, 0x5c, 0xb1, 0xab, 0x9e, 0x9c, 0x52, 0x1b, 0x5f, 0x93, 0x0a, 0xef, + 0x91, 0x85, 0x49, 0xee, 0x2d, 0x4f, 0x8f, 0x3b, 0x47, 0x87, 0x6d, 0x46, 0xd6, 0x3e, 0x69, 0x64, + 0x2a, 0xce, 0xcb, 0x2f, 0xfc, 0x97, 0x05, 0x7a, 0xac, 0x7f, 0xd5, 0x1a, 0x4b, 0x0e, 0xa7, 0x5a, + 0x28, 0x14, 0x3f, 0x29, 0x88, 0x3c, 0x4c, 0x02, 0xb8, 0xda, 0xb0, 0x17, 0x55, 0x1f, 0x8a, 0x7d, + 0x57, 0xc7, 0x8d, 0x74, 0xb7, 0xc4, 0x9f, 0x72, 0x7e, 0x15, 0x22, 0x12, 0x58, 0x07, 0x99, 0x34, + 0x6e, 0x50, 0xde, 0x68, 0x65, 0xbc, 0xdb, 0xf8, 0xc8, 0xa8, 0x2b, 0x40, 0xdc, 0xfe, 0x32, 0xa4, + 0xca, 0x10, 0x21, 0xf0, 0xd3, 0x5d, 0x0f, 0x00, 0x6f, 0x9d, 0x36, 0x42, 0x4a, 0x5e, 0xc1, 0xe0, + }, + { + 0x75, 0xf3, 0xc6, 0xf4, 0xdb, 0x7b, 0xfb, 0xc8, 0x4a, 0xd3, 0xe6, 0x6b, 0x45, 0x7d, 0xe8, 0x4b, + 0xd6, 0x32, 0xd8, 0xfd, 0x37, 0x71, 0xf1, 0xe1, 0x30, 0x0f, 0xf8, 0x1b, 0x87, 0xfa, 0x06, 0x3f, + 0x5e, 0xba, 0xae, 0x5b, 0x8a, 0x00, 0xbc, 0x9d, 0x6d, 0xc1, 0xb1, 0x0e, 0x80, 0x5d, 0xd2, 0xd5, + 0xa0, 0x84, 0x07, 0x14, 0xb5, 0x90, 0x2c, 0xa3, 0xb2, 0x73, 0x4c, 0x54, 0x92, 0x74, 0x36, 0x51, + 0x38, 0xb0, 0xbd, 0x5a, 0xfc, 0x60, 0x62, 0x96, 0x6c, 0x42, 0xf7, 0x10, 0x7c, 0x28, 0x27, 0x8c, + 0x13, 0x95, 0x9c, 0xc7, 0x24, 0x46, 0x3b, 0x70, 0xca, 0xe3, 0x85, 0xcb, 0x11, 0xd0, 0x93, 0xb8, + 0xa6, 0x83, 0x20, 0xff, 0x9f, 0x77, 0xc3, 0xcc, 0x03, 0x6f, 0x08, 0xbf, 0x40, 0xe7, 0x2b, 0xe2, + 0x79, 0x0c, 0xaa, 0x82, 0x41, 0x3a, 0xea, 0xb9, 0xe4, 0x9a, 0xa4, 0x97, 0x7e, 0xda, 0x7a, 0x17, + 0x66, 0x94, 0xa1, 0x1d, 0x3d, 0xf0, 0xde, 0xb3, 0x0b, 0x72, 0xa7, 0x1c, 0xef, 0xd1, 0x53, 0x3e, + 0x8f, 0x33, 0x26, 0x5f, 0xec, 0x76, 0x2a, 0x49, 0x81, 0x88, 0xee, 0x21, 0xc4, 0x1a, 0xeb, 0xd9, + 0xc5, 0x39, 0x99, 0xcd, 0xad, 0x31, 0x8b, 0x01, 0x18, 0x23, 0xdd, 0x1f, 0x4e, 0x2d, 0xf9, 0x48, + 0x4f, 0xf2, 0x65, 0x8e, 0x78, 0x5c, 0x58, 0x19, 0x8d, 0xe5, 0x98, 0x57, 0x67, 0x7f, 0x05, 0x64, + 0xaf, 0x63, 0xb6, 0xfe, 0xf5, 0xb7, 0x3c, 0xa5, 0xce, 0xe9, 0x68, 0x44, 0xe0, 0x4d, 0x43, 0x69, + 0x29, 0x2e, 0xac, 0x15, 0x59, 0xa8, 0x0a, 0x9e, 0x6e, 0x47, 0xdf, 0x34, 0x35, 0x6a, 0xcf, 0xdc, + 0x22, 0xc9, 0xc0, 0x9b, 0x89, 0xd4, 0xed, 0xab, 0x12, 0xa2, 0x0d, 0x52, 0xbb, 0x02, 0x2f, 0xa9, + 0xd7, 0x61, 0x1e, 0xb4, 0x50, 0x04, 0xf6, 0xc2, 0x16, 0x25, 0x86, 0x56, 0x55, 0x09, 0xbe, 0x91, + }, +} + +// gfMult returns a·b in GF(2^8)/p +func gfMult(a, b byte, p uint32) byte { + B := [2]uint32{0, uint32(b)} + P := [2]uint32{0, p} + var result uint32 + + // branchless GF multiplier + for i := 0; i < 7; i++ { + result ^= B[a&1] + a >>= 1 + B[1] = P[B[1]>>7] ^ (B[1] << 1) + } + result ^= B[a&1] + return byte(result) +} + +// mdsColumnMult calculates y{col} where [y0 y1 y2 y3] = MDS · [x0] +func mdsColumnMult(in byte, col int) uint32 { + mul01 := in + mul5B := gfMult(in, 0x5B, mdsPolynomial) + mulEF := gfMult(in, 0xEF, mdsPolynomial) + + switch col { + case 0: + return uint32(mul01) | uint32(mul5B)<<8 | uint32(mulEF)<<16 | uint32(mulEF)<<24 + case 1: + return uint32(mulEF) | uint32(mulEF)<<8 | uint32(mul5B)<<16 | uint32(mul01)<<24 + case 2: + return uint32(mul5B) | uint32(mulEF)<<8 | uint32(mul01)<<16 | uint32(mulEF)<<24 + case 3: + return uint32(mul5B) | uint32(mul01)<<8 | uint32(mulEF)<<16 | uint32(mul5B)<<24 + } + + panic("unreachable") +} + +// h implements the S-box generation function. See [TWOFISH] 4.3.5 +func h(in, key []byte, offset int) uint32 { + var y [4]byte + for x := range y { + y[x] = in[x] + } + switch len(key) / 8 { + case 4: + y[0] = sbox[1][y[0]] ^ key[4*(6+offset)+0] + y[1] = sbox[0][y[1]] ^ key[4*(6+offset)+1] + y[2] = sbox[0][y[2]] ^ key[4*(6+offset)+2] + y[3] = sbox[1][y[3]] ^ key[4*(6+offset)+3] + fallthrough + case 3: + y[0] = sbox[1][y[0]] ^ key[4*(4+offset)+0] + y[1] = sbox[1][y[1]] ^ key[4*(4+offset)+1] + y[2] = sbox[0][y[2]] ^ key[4*(4+offset)+2] + y[3] = sbox[0][y[3]] ^ key[4*(4+offset)+3] + fallthrough + case 2: + y[0] = sbox[1][sbox[0][sbox[0][y[0]]^key[4*(2+offset)+0]]^key[4*(0+offset)+0]] + y[1] = sbox[0][sbox[0][sbox[1][y[1]]^key[4*(2+offset)+1]]^key[4*(0+offset)+1]] + y[2] = sbox[1][sbox[1][sbox[0][y[2]]^key[4*(2+offset)+2]]^key[4*(0+offset)+2]] + y[3] = sbox[0][sbox[1][sbox[1][y[3]]^key[4*(2+offset)+3]]^key[4*(0+offset)+3]] + } + // [y0 y1 y2 y3] = MDS . [x0 x1 x2 x3] + var mdsMult uint32 + for i := range y { + mdsMult ^= mdsColumnMult(y[i], i) + } + return mdsMult +} + +// Encrypt encrypts a 16-byte block from src to dst, which may overlap. +// Note that for amounts of data larger than a block, +// it is not safe to just call Encrypt on successive blocks; +// instead, use an encryption mode like CBC (see crypto/block/cbc.go). +func (c *Cipher) Encrypt(dst, src []byte) { + S1 := c.s[0] + S2 := c.s[1] + S3 := c.s[2] + S4 := c.s[3] + + // Load input + ia := load32l(src[0:4]) + ib := load32l(src[4:8]) + ic := load32l(src[8:12]) + id := load32l(src[12:16]) + + // Pre-whitening + ia ^= c.k[0] + ib ^= c.k[1] + ic ^= c.k[2] + id ^= c.k[3] + + for i := 0; i < 8; i++ { + k := c.k[8+i*4 : 12+i*4] + t2 := S2[byte(ib)] ^ S3[byte(ib>>8)] ^ S4[byte(ib>>16)] ^ S1[byte(ib>>24)] + t1 := S1[byte(ia)] ^ S2[byte(ia>>8)] ^ S3[byte(ia>>16)] ^ S4[byte(ia>>24)] + t2 + ic = ror(ic^(t1+k[0]), 1) + id = rol(id, 1) ^ (t2 + t1 + k[1]) + + t2 = S2[byte(id)] ^ S3[byte(id>>8)] ^ S4[byte(id>>16)] ^ S1[byte(id>>24)] + t1 = S1[byte(ic)] ^ S2[byte(ic>>8)] ^ S3[byte(ic>>16)] ^ S4[byte(ic>>24)] + t2 + ia = ror(ia^(t1+k[2]), 1) + ib = rol(ib, 1) ^ (t2 + t1 + k[3]) + } + + // Output with "undo last swap" + ta := ic ^ c.k[4] + tb := id ^ c.k[5] + tc := ia ^ c.k[6] + td := ib ^ c.k[7] + + store32l(dst[0:4], ta) + store32l(dst[4:8], tb) + store32l(dst[8:12], tc) + store32l(dst[12:16], td) +} + +// Decrypt decrypts a 16-byte block from src to dst, which may overlap. +func (c *Cipher) Decrypt(dst, src []byte) { + S1 := c.s[0] + S2 := c.s[1] + S3 := c.s[2] + S4 := c.s[3] + + // Load input + ta := load32l(src[0:4]) + tb := load32l(src[4:8]) + tc := load32l(src[8:12]) + td := load32l(src[12:16]) + + // Undo undo final swap + ia := tc ^ c.k[6] + ib := td ^ c.k[7] + ic := ta ^ c.k[4] + id := tb ^ c.k[5] + + for i := 8; i > 0; i-- { + k := c.k[4+i*4 : 8+i*4] + t2 := S2[byte(id)] ^ S3[byte(id>>8)] ^ S4[byte(id>>16)] ^ S1[byte(id>>24)] + t1 := S1[byte(ic)] ^ S2[byte(ic>>8)] ^ S3[byte(ic>>16)] ^ S4[byte(ic>>24)] + t2 + ia = rol(ia, 1) ^ (t1 + k[2]) + ib = ror(ib^(t2+t1+k[3]), 1) + + t2 = S2[byte(ib)] ^ S3[byte(ib>>8)] ^ S4[byte(ib>>16)] ^ S1[byte(ib>>24)] + t1 = S1[byte(ia)] ^ S2[byte(ia>>8)] ^ S3[byte(ia>>16)] ^ S4[byte(ia>>24)] + t2 + ic = rol(ic, 1) ^ (t1 + k[0]) + id = ror(id^(t2+t1+k[1]), 1) + } + + // Undo pre-whitening + ia ^= c.k[0] + ib ^= c.k[1] + ic ^= c.k[2] + id ^= c.k[3] + + store32l(dst[0:4], ia) + store32l(dst[4:8], ib) + store32l(dst[8:12], ic) + store32l(dst[12:16], id) +} diff --git a/libgo/go/crypto/twofish/twofish_test.go b/libgo/go/crypto/twofish/twofish_test.go new file mode 100644 index 000000000..303081f3f --- /dev/null +++ b/libgo/go/crypto/twofish/twofish_test.go @@ -0,0 +1,129 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package twofish + +import ( + "bytes" + "testing" +) + +var qbox = [2][4][16]byte{ + { + {0x8, 0x1, 0x7, 0xD, 0x6, 0xF, 0x3, 0x2, 0x0, 0xB, 0x5, 0x9, 0xE, 0xC, 0xA, 0x4}, + {0xE, 0xC, 0xB, 0x8, 0x1, 0x2, 0x3, 0x5, 0xF, 0x4, 0xA, 0x6, 0x7, 0x0, 0x9, 0xD}, + {0xB, 0xA, 0x5, 0xE, 0x6, 0xD, 0x9, 0x0, 0xC, 0x8, 0xF, 0x3, 0x2, 0x4, 0x7, 0x1}, + {0xD, 0x7, 0xF, 0x4, 0x1, 0x2, 0x6, 0xE, 0x9, 0xB, 0x3, 0x0, 0x8, 0x5, 0xC, 0xA}, + }, + { + {0x2, 0x8, 0xB, 0xD, 0xF, 0x7, 0x6, 0xE, 0x3, 0x1, 0x9, 0x4, 0x0, 0xA, 0xC, 0x5}, + {0x1, 0xE, 0x2, 0xB, 0x4, 0xC, 0x3, 0x7, 0x6, 0xD, 0xA, 0x5, 0xF, 0x9, 0x0, 0x8}, + {0x4, 0xC, 0x7, 0x5, 0x1, 0x6, 0x9, 0xA, 0x0, 0xE, 0xD, 0x8, 0x2, 0xB, 0x3, 0xF}, + {0xB, 0x9, 0x5, 0x1, 0xC, 0x3, 0xD, 0xE, 0x6, 0x4, 0x7, 0xF, 0x2, 0x0, 0x8, 0xA}, + }, +} + +// genSbox generates the variable sbox +func genSbox(qi int, x byte) byte { + a0, b0 := x/16, x%16 + for i := 0; i < 2; i++ { + a1 := a0 ^ b0 + b1 := (a0 ^ ((b0 << 3) | (b0 >> 1)) ^ (a0 << 3)) & 15 + a0 = qbox[qi][2*i][a1] + b0 = qbox[qi][2*i+1][b1] + } + return (b0 << 4) + a0 +} + +func TestSbox(t *testing.T) { + for n := range sbox { + for m := range sbox[n] { + if genSbox(n, byte(m)) != sbox[n][m] { + t.Errorf("#%d|%d: sbox value = %d want %d", n, m, sbox[n][m], genSbox(n, byte(m))) + } + } + } +} + +var testVectors = []struct { + key []byte + dec []byte + enc []byte +}{ + // These tests are extracted from LibTom + { + []byte{0x9F, 0x58, 0x9F, 0x5C, 0xF6, 0x12, 0x2C, 0x32, 0xB6, 0xBF, 0xEC, 0x2F, 0x2A, 0xE8, 0xC3, 0x5A}, + []byte{0xD4, 0x91, 0xDB, 0x16, 0xE7, 0xB1, 0xC3, 0x9E, 0x86, 0xCB, 0x08, 0x6B, 0x78, 0x9F, 0x54, 0x19}, + []byte{0x01, 0x9F, 0x98, 0x09, 0xDE, 0x17, 0x11, 0x85, 0x8F, 0xAA, 0xC3, 0xA3, 0xBA, 0x20, 0xFB, 0xC3}, + }, + { + []byte{0x88, 0xB2, 0xB2, 0x70, 0x6B, 0x10, 0x5E, 0x36, 0xB4, 0x46, 0xBB, 0x6D, 0x73, 0x1A, 0x1E, 0x88, + 0xEF, 0xA7, 0x1F, 0x78, 0x89, 0x65, 0xBD, 0x44}, + []byte{0x39, 0xDA, 0x69, 0xD6, 0xBA, 0x49, 0x97, 0xD5, 0x85, 0xB6, 0xDC, 0x07, 0x3C, 0xA3, 0x41, 0xB2}, + []byte{0x18, 0x2B, 0x02, 0xD8, 0x14, 0x97, 0xEA, 0x45, 0xF9, 0xDA, 0xAC, 0xDC, 0x29, 0x19, 0x3A, 0x65}, + }, + { + []byte{0xD4, 0x3B, 0xB7, 0x55, 0x6E, 0xA3, 0x2E, 0x46, 0xF2, 0xA2, 0x82, 0xB7, 0xD4, 0x5B, 0x4E, 0x0D, + 0x57, 0xFF, 0x73, 0x9D, 0x4D, 0xC9, 0x2C, 0x1B, 0xD7, 0xFC, 0x01, 0x70, 0x0C, 0xC8, 0x21, 0x6F}, + []byte{0x90, 0xAF, 0xE9, 0x1B, 0xB2, 0x88, 0x54, 0x4F, 0x2C, 0x32, 0xDC, 0x23, 0x9B, 0x26, 0x35, 0xE6}, + []byte{0x6C, 0xB4, 0x56, 0x1C, 0x40, 0xBF, 0x0A, 0x97, 0x05, 0x93, 0x1C, 0xB6, 0xD4, 0x08, 0xE7, 0xFA}, + }, + // These test are derived from http://www.schneier.com/code/ecb_ival.txt + { + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x9F, 0x58, 0x9F, 0x5C, 0xF6, 0x12, 0x2C, 0x32, 0xB6, 0xBF, 0xEC, 0x2F, 0x2A, 0xE8, 0xC3, 0x5A}, + }, + { + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF, 0xFE, 0xDC, 0xBA, 0x98, 0x76, 0x54, 0x32, 0x10, + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, + }, + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xCF, 0xD1, 0xD2, 0xE5, 0xA9, 0xBE, 0x9C, 0xDF, 0x50, 0x1F, 0x13, 0xB8, 0x92, 0xBD, 0x22, 0x48}, + }, + { + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF, 0xFE, 0xDC, 0xBA, 0x98, 0x76, 0x54, 0x32, 0x10, + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF, + }, + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x37, 0x52, 0x7B, 0xE0, 0x05, 0x23, 0x34, 0xB8, 0x9F, 0x0C, 0xFC, 0xCA, 0xE8, 0x7C, 0xFA, 0x20}, + }, +} + +func TestCipher(t *testing.T) { + for n, tt := range testVectors { + // Test if the plaintext (dec) is encrypts to the given + // ciphertext (enc) using the given key. Test also if enc can + // be decrypted again into dec. + c, err := NewCipher(tt.key) + if err != nil { + t.Errorf("#%d: NewCipher: %v", n, err) + return + } + + buf := make([]byte, 16) + c.Encrypt(buf, tt.dec) + if !bytes.Equal(buf, tt.enc) { + t.Errorf("#%d: encrypt = %x want %x", n, buf, tt.enc) + } + c.Decrypt(buf, tt.enc) + if !bytes.Equal(buf, tt.dec) { + t.Errorf("#%d: decrypt = %x want %x", n, buf, tt.dec) + } + + // Test that 16 zero bytes, encrypted 1000 times then decrypted + // 1000 times results in zero bytes again. + zero := make([]byte, 16) + buf = make([]byte, 16) + for i := 0; i < 1000; i++ { + c.Encrypt(buf, buf) + } + for i := 0; i < 1000; i++ { + c.Decrypt(buf, buf) + } + if !bytes.Equal(buf, zero) { + t.Errorf("#%d: encrypt/decrypt 1000: have %x want %x", n, buf, zero) + } + } +} diff --git a/libgo/go/crypto/x509/x509.go b/libgo/go/crypto/x509/x509.go new file mode 100644 index 000000000..6199e8db9 --- /dev/null +++ b/libgo/go/crypto/x509/x509.go @@ -0,0 +1,854 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This package parses X.509-encoded keys and certificates. +package x509 + +import ( + "asn1" + "big" + "container/vector" + "crypto/rsa" + "crypto/sha1" + "hash" + "io" + "os" + "strings" + "time" +) + +// pkcs1PrivateKey is a structure which mirrors the PKCS#1 ASN.1 for an RSA private key. +type pkcs1PrivateKey struct { + Version int + N asn1.RawValue + E int + D asn1.RawValue + P asn1.RawValue + Q asn1.RawValue +} + +// rawValueIsInteger returns true iff the given ASN.1 RawValue is an INTEGER type. +func rawValueIsInteger(raw *asn1.RawValue) bool { + return raw.Class == 0 && raw.Tag == 2 && raw.IsCompound == false +} + +// ParsePKCS1PrivateKey returns an RSA private key from its ASN.1 PKCS#1 DER encoded form. +func ParsePKCS1PrivateKey(der []byte) (key *rsa.PrivateKey, err os.Error) { + var priv pkcs1PrivateKey + rest, err := asn1.Unmarshal(der, &priv) + if len(rest) > 0 { + err = asn1.SyntaxError{"trailing data"} + return + } + if err != nil { + return + } + + if !rawValueIsInteger(&priv.N) || + !rawValueIsInteger(&priv.D) || + !rawValueIsInteger(&priv.P) || + !rawValueIsInteger(&priv.Q) { + err = asn1.StructuralError{"tags don't match"} + return + } + + key = &rsa.PrivateKey{ + PublicKey: rsa.PublicKey{ + E: priv.E, + N: new(big.Int).SetBytes(priv.N.Bytes), + }, + D: new(big.Int).SetBytes(priv.D.Bytes), + P: new(big.Int).SetBytes(priv.P.Bytes), + Q: new(big.Int).SetBytes(priv.Q.Bytes), + } + + err = key.Validate() + if err != nil { + return nil, err + } + return +} + +// MarshalPKCS1PrivateKey converts a private key to ASN.1 DER encoded form. +func MarshalPKCS1PrivateKey(key *rsa.PrivateKey) []byte { + priv := pkcs1PrivateKey{ + Version: 1, + N: asn1.RawValue{Tag: 2, Bytes: key.PublicKey.N.Bytes()}, + E: key.PublicKey.E, + D: asn1.RawValue{Tag: 2, Bytes: key.D.Bytes()}, + P: asn1.RawValue{Tag: 2, Bytes: key.P.Bytes()}, + Q: asn1.RawValue{Tag: 2, Bytes: key.Q.Bytes()}, + } + + b, _ := asn1.Marshal(priv) + return b +} + +// These structures reflect the ASN.1 structure of X.509 certificates.: + +type certificate struct { + TBSCertificate tbsCertificate + SignatureAlgorithm algorithmIdentifier + SignatureValue asn1.BitString +} + +type tbsCertificate struct { + Raw asn1.RawContent + Version int "optional,explicit,default:1,tag:0" + SerialNumber asn1.RawValue + SignatureAlgorithm algorithmIdentifier + Issuer rdnSequence + Validity validity + Subject rdnSequence + PublicKey publicKeyInfo + UniqueId asn1.BitString "optional,tag:1" + SubjectUniqueId asn1.BitString "optional,tag:2" + Extensions []extension "optional,explicit,tag:3" +} + +type algorithmIdentifier struct { + Algorithm asn1.ObjectIdentifier +} + +type rdnSequence []relativeDistinguishedNameSET + +type relativeDistinguishedNameSET []attributeTypeAndValue + +type attributeTypeAndValue struct { + Type asn1.ObjectIdentifier + Value interface{} +} + +type validity struct { + NotBefore, NotAfter *time.Time +} + +type publicKeyInfo struct { + Algorithm algorithmIdentifier + PublicKey asn1.BitString +} + +type extension struct { + Id asn1.ObjectIdentifier + Critical bool "optional" + Value []byte +} + +// RFC 5280, 4.2.1.1 +type authKeyId struct { + Id []byte "optional,tag:0" +} + +type SignatureAlgorithm int + +const ( + UnknownSignatureAlgorithm SignatureAlgorithm = iota + MD2WithRSA + MD5WithRSA + SHA1WithRSA + SHA256WithRSA + SHA384WithRSA + SHA512WithRSA +) + +type PublicKeyAlgorithm int + +const ( + UnknownPublicKeyAlgorithm PublicKeyAlgorithm = iota + RSA +) + +// Name represents an X.509 distinguished name. This only includes the common +// elements of a DN. Additional elements in the name are ignored. +type Name struct { + Country, Organization, OrganizationalUnit []string + Locality, Province []string + StreetAddress, PostalCode []string + SerialNumber, CommonName string +} + +func (n *Name) fillFromRDNSequence(rdns *rdnSequence) { + for _, rdn := range *rdns { + if len(rdn) == 0 { + continue + } + atv := rdn[0] + value, ok := atv.Value.(string) + if !ok { + continue + } + + t := atv.Type + if len(t) == 4 && t[0] == 2 && t[1] == 5 && t[2] == 4 { + switch t[3] { + case 3: + n.CommonName = value + case 5: + n.SerialNumber = value + case 6: + n.Country = append(n.Country, value) + case 7: + n.Locality = append(n.Locality, value) + case 8: + n.Province = append(n.Province, value) + case 9: + n.StreetAddress = append(n.StreetAddress, value) + case 10: + n.Organization = append(n.Organization, value) + case 11: + n.OrganizationalUnit = append(n.OrganizationalUnit, value) + case 17: + n.PostalCode = append(n.PostalCode, value) + } + } + } +} + +var ( + oidCountry = []int{2, 5, 4, 6} + oidOrganization = []int{2, 5, 4, 10} + oidOrganizationalUnit = []int{2, 5, 4, 11} + oidCommonName = []int{2, 5, 4, 3} + oidSerialNumber = []int{2, 5, 4, 5} + oidLocatity = []int{2, 5, 4, 7} + oidProvince = []int{2, 5, 4, 8} + oidStreetAddress = []int{2, 5, 4, 9} + oidPostalCode = []int{2, 5, 4, 17} +) + +// appendRDNs appends a relativeDistinguishedNameSET to the given rdnSequence +// and returns the new value. The relativeDistinguishedNameSET contains an +// attributeTypeAndValue for each of the given values. See RFC 5280, A.1, and +// search for AttributeTypeAndValue. +func appendRDNs(in rdnSequence, values []string, oid asn1.ObjectIdentifier) rdnSequence { + if len(values) == 0 { + return in + } + + s := make([]attributeTypeAndValue, len(values)) + for i, value := range values { + s[i].Type = oid + s[i].Value = value + } + + return append(in, s) +} + +func (n Name) toRDNSequence() (ret rdnSequence) { + ret = appendRDNs(ret, n.Country, oidCountry) + ret = appendRDNs(ret, n.Organization, oidOrganization) + ret = appendRDNs(ret, n.OrganizationalUnit, oidOrganizationalUnit) + ret = appendRDNs(ret, n.Locality, oidLocatity) + ret = appendRDNs(ret, n.Province, oidProvince) + ret = appendRDNs(ret, n.StreetAddress, oidStreetAddress) + ret = appendRDNs(ret, n.PostalCode, oidPostalCode) + if len(n.CommonName) > 0 { + ret = appendRDNs(ret, []string{n.CommonName}, oidCommonName) + } + if len(n.SerialNumber) > 0 { + ret = appendRDNs(ret, []string{n.SerialNumber}, oidSerialNumber) + } + + return ret +} + +func getSignatureAlgorithmFromOID(oid []int) SignatureAlgorithm { + if len(oid) == 7 && oid[0] == 1 && oid[1] == 2 && oid[2] == 840 && + oid[3] == 113549 && oid[4] == 1 && oid[5] == 1 { + switch oid[6] { + case 2: + return MD2WithRSA + case 4: + return MD5WithRSA + case 5: + return SHA1WithRSA + case 11: + return SHA256WithRSA + case 12: + return SHA384WithRSA + case 13: + return SHA512WithRSA + } + } + + return UnknownSignatureAlgorithm +} + +func getPublicKeyAlgorithmFromOID(oid []int) PublicKeyAlgorithm { + if len(oid) == 7 && oid[0] == 1 && oid[1] == 2 && oid[2] == 840 && + oid[3] == 113549 && oid[4] == 1 && oid[5] == 1 { + switch oid[6] { + case 1: + return RSA + } + } + + return UnknownPublicKeyAlgorithm +} + +// KeyUsage represents the set of actions that are valid for a given key. It's +// a bitmap of the KeyUsage* constants. +type KeyUsage int + +const ( + KeyUsageDigitalSignature KeyUsage = 1 << iota + KeyUsageContentCommitment + KeyUsageKeyEncipherment + KeyUsageDataEncipherment + KeyUsageKeyAgreement + KeyUsageCertSign + KeyUsageCRLSign + KeyUsageEncipherOnly + KeyUsageDecipherOnly +) + +// A Certificate represents an X.509 certificate. +type Certificate struct { + Raw []byte // Raw ASN.1 DER contents. + Signature []byte + SignatureAlgorithm SignatureAlgorithm + + PublicKeyAlgorithm PublicKeyAlgorithm + PublicKey interface{} + + Version int + SerialNumber []byte + Issuer Name + Subject Name + NotBefore, NotAfter *time.Time // Validity bounds. + KeyUsage KeyUsage + + BasicConstraintsValid bool // if true then the next two fields are valid. + IsCA bool + MaxPathLen int + + SubjectKeyId []byte + AuthorityKeyId []byte + + // Subject Alternate Name values + DNSNames []string + EmailAddresses []string + + PolicyIdentifiers []asn1.ObjectIdentifier +} + +// UnsupportedAlgorithmError results from attempting to perform an operation +// that involves algorithms that are not currently implemented. +type UnsupportedAlgorithmError struct{} + +func (UnsupportedAlgorithmError) String() string { + return "cannot verify signature: algorithm unimplemented" +} + +// ConstraintViolationError results when a requested usage is not permitted by +// a certificate. For example: checking a signature when the public key isn't a +// certificate signing key. +type ConstraintViolationError struct{} + +func (ConstraintViolationError) String() string { + return "invalid signature: parent certificate cannot sign this kind of certificate" +} + +// CheckSignatureFrom verifies that the signature on c is a valid signature +// from parent. +func (c *Certificate) CheckSignatureFrom(parent *Certificate) (err os.Error) { + // RFC 5280, 4.2.1.9: + // "If the basic constraints extension is not present in a version 3 + // certificate, or the extension is present but the cA boolean is not + // asserted, then the certified public key MUST NOT be used to verify + // certificate signatures." + if parent.Version == 3 && !parent.BasicConstraintsValid || + parent.BasicConstraintsValid && !parent.IsCA { + return ConstraintViolationError{} + } + + if parent.KeyUsage != 0 && parent.KeyUsage&KeyUsageCertSign == 0 { + return ConstraintViolationError{} + } + + if parent.PublicKeyAlgorithm == UnknownPublicKeyAlgorithm { + return UnsupportedAlgorithmError{} + } + + // TODO(agl): don't ignore the path length constraint. + + var h hash.Hash + var hashType rsa.PKCS1v15Hash + + switch c.SignatureAlgorithm { + case SHA1WithRSA: + h = sha1.New() + hashType = rsa.HashSHA1 + default: + return UnsupportedAlgorithmError{} + } + + pub, ok := parent.PublicKey.(*rsa.PublicKey) + if !ok { + return UnsupportedAlgorithmError{} + } + + h.Write(c.Raw) + digest := h.Sum() + + return rsa.VerifyPKCS1v15(pub, hashType, digest, c.Signature) +} + +func matchHostnames(pattern, host string) bool { + if len(pattern) == 0 || len(host) == 0 { + return false + } + + patternParts := strings.Split(pattern, ".", -1) + hostParts := strings.Split(host, ".", -1) + + if len(patternParts) != len(hostParts) { + return false + } + + for i, patternPart := range patternParts { + if patternPart == "*" { + continue + } + if patternPart != hostParts[i] { + return false + } + } + + return true +} + +type HostnameError struct { + Certificate *Certificate + Host string +} + +func (h *HostnameError) String() string { + var valid string + c := h.Certificate + if len(c.DNSNames) > 0 { + valid = strings.Join(c.DNSNames, ", ") + } else { + valid = c.Subject.CommonName + } + return "certificate is valid for " + valid + ", not " + h.Host +} + +// VerifyHostname returns nil if c is a valid certificate for the named host. +// Otherwise it returns an os.Error describing the mismatch. +func (c *Certificate) VerifyHostname(h string) os.Error { + if len(c.DNSNames) > 0 { + for _, match := range c.DNSNames { + if matchHostnames(match, h) { + return nil + } + } + // If Subject Alt Name is given, we ignore the common name. + } else if matchHostnames(c.Subject.CommonName, h) { + return nil + } + + return &HostnameError{c, h} +} + +type UnhandledCriticalExtension struct{} + +func (h UnhandledCriticalExtension) String() string { + return "unhandled critical extension" +} + +type basicConstraints struct { + IsCA bool "optional" + MaxPathLen int "optional" +} + +type rsaPublicKey struct { + N asn1.RawValue + E int +} + +// RFC 5280 4.2.1.4 +type policyInformation struct { + Policy asn1.ObjectIdentifier + // policyQualifiers omitted +} + +func parsePublicKey(algo PublicKeyAlgorithm, asn1Data []byte) (interface{}, os.Error) { + switch algo { + case RSA: + p := new(rsaPublicKey) + _, err := asn1.Unmarshal(asn1Data, p) + if err != nil { + return nil, err + } + + if !rawValueIsInteger(&p.N) { + return nil, asn1.StructuralError{"tags don't match"} + } + + pub := &rsa.PublicKey{ + E: p.E, + N: new(big.Int).SetBytes(p.N.Bytes), + } + return pub, nil + default: + return nil, nil + } + + panic("unreachable") +} + +func parseCertificate(in *certificate) (*Certificate, os.Error) { + out := new(Certificate) + out.Raw = in.TBSCertificate.Raw + + out.Signature = in.SignatureValue.RightAlign() + out.SignatureAlgorithm = + getSignatureAlgorithmFromOID(in.TBSCertificate.SignatureAlgorithm.Algorithm) + + out.PublicKeyAlgorithm = + getPublicKeyAlgorithmFromOID(in.TBSCertificate.PublicKey.Algorithm.Algorithm) + var err os.Error + out.PublicKey, err = parsePublicKey(out.PublicKeyAlgorithm, in.TBSCertificate.PublicKey.PublicKey.RightAlign()) + if err != nil { + return nil, err + } + + out.Version = in.TBSCertificate.Version + 1 + out.SerialNumber = in.TBSCertificate.SerialNumber.Bytes + out.Issuer.fillFromRDNSequence(&in.TBSCertificate.Issuer) + out.Subject.fillFromRDNSequence(&in.TBSCertificate.Subject) + out.NotBefore = in.TBSCertificate.Validity.NotBefore + out.NotAfter = in.TBSCertificate.Validity.NotAfter + + for _, e := range in.TBSCertificate.Extensions { + if len(e.Id) == 4 && e.Id[0] == 2 && e.Id[1] == 5 && e.Id[2] == 29 { + switch e.Id[3] { + case 15: + // RFC 5280, 4.2.1.3 + var usageBits asn1.BitString + _, err := asn1.Unmarshal(e.Value, &usageBits) + + if err == nil { + var usage int + for i := 0; i < 9; i++ { + if usageBits.At(i) != 0 { + usage |= 1 << uint(i) + } + } + out.KeyUsage = KeyUsage(usage) + continue + } + case 19: + // RFC 5280, 4.2.1.9 + var constriants basicConstraints + _, err := asn1.Unmarshal(e.Value, &constriants) + + if err == nil { + out.BasicConstraintsValid = true + out.IsCA = constriants.IsCA + out.MaxPathLen = constriants.MaxPathLen + continue + } + case 17: + // RFC 5280, 4.2.1.6 + + // SubjectAltName ::= GeneralNames + // + // GeneralNames ::= SEQUENCE SIZE (1..MAX) OF GeneralName + // + // GeneralName ::= CHOICE { + // otherName [0] OtherName, + // rfc822Name [1] IA5String, + // dNSName [2] IA5String, + // x400Address [3] ORAddress, + // directoryName [4] Name, + // ediPartyName [5] EDIPartyName, + // uniformResourceIdentifier [6] IA5String, + // iPAddress [7] OCTET STRING, + // registeredID [8] OBJECT IDENTIFIER } + var seq asn1.RawValue + _, err := asn1.Unmarshal(e.Value, &seq) + if err != nil { + return nil, err + } + if !seq.IsCompound || seq.Tag != 16 || seq.Class != 0 { + return nil, asn1.StructuralError{"bad SAN sequence"} + } + + parsedName := false + + rest := seq.Bytes + for len(rest) > 0 { + var v asn1.RawValue + rest, err = asn1.Unmarshal(rest, &v) + if err != nil { + return nil, err + } + switch v.Tag { + case 1: + out.EmailAddresses = append(out.EmailAddresses, string(v.Bytes)) + parsedName = true + case 2: + out.DNSNames = append(out.DNSNames, string(v.Bytes)) + parsedName = true + } + } + + if parsedName { + continue + } + // If we didn't parse any of the names then we + // fall through to the critical check below. + + case 35: + // RFC 5280, 4.2.1.1 + var a authKeyId + _, err = asn1.Unmarshal(e.Value, &a) + if err != nil { + return nil, err + } + out.AuthorityKeyId = a.Id + continue + + case 14: + // RFC 5280, 4.2.1.2 + var keyid []byte + _, err = asn1.Unmarshal(e.Value, &keyid) + if err != nil { + return nil, err + } + out.SubjectKeyId = keyid + continue + + case 32: + // RFC 5280 4.2.1.4: Certificate Policies + var policies []policyInformation + if _, err = asn1.Unmarshal(e.Value, &policies); err != nil { + return nil, err + } + out.PolicyIdentifiers = make([]asn1.ObjectIdentifier, len(policies)) + for i, policy := range policies { + out.PolicyIdentifiers[i] = policy.Policy + } + } + } + + if e.Critical { + return out, UnhandledCriticalExtension{} + } + } + + return out, nil +} + +// ParseCertificate parses a single certificate from the given ASN.1 DER data. +func ParseCertificate(asn1Data []byte) (*Certificate, os.Error) { + var cert certificate + rest, err := asn1.Unmarshal(asn1Data, &cert) + if err != nil { + return nil, err + } + if len(rest) > 0 { + return nil, asn1.SyntaxError{"trailing data"} + } + + return parseCertificate(&cert) +} + +// ParseCertificates parses one or more certificates from the given ASN.1 DER +// data. The certificates must be concatenated with no intermediate padding. +func ParseCertificates(asn1Data []byte) ([]*Certificate, os.Error) { + v := new(vector.Vector) + + for len(asn1Data) > 0 { + cert := new(certificate) + var err os.Error + asn1Data, err = asn1.Unmarshal(asn1Data, cert) + if err != nil { + return nil, err + } + v.Push(cert) + } + + ret := make([]*Certificate, v.Len()) + for i := 0; i < v.Len(); i++ { + cert, err := parseCertificate(v.At(i).(*certificate)) + if err != nil { + return nil, err + } + ret[i] = cert + } + + return ret, nil +} + +func reverseBitsInAByte(in byte) byte { + b1 := in>>4 | in<<4 + b2 := b1>>2&0x33 | b1<<2&0xcc + b3 := b2>>1&0x55 | b2<<1&0xaa + return b3 +} + +var ( + oidExtensionSubjectKeyId = []int{2, 5, 29, 14} + oidExtensionKeyUsage = []int{2, 5, 29, 15} + oidExtensionAuthorityKeyId = []int{2, 5, 29, 35} + oidExtensionBasicConstraints = []int{2, 5, 29, 19} + oidExtensionSubjectAltName = []int{2, 5, 29, 17} + oidExtensionCertificatePolicies = []int{2, 5, 29, 32} +) + +func buildExtensions(template *Certificate) (ret []extension, err os.Error) { + ret = make([]extension, 6 /* maximum number of elements. */ ) + n := 0 + + if template.KeyUsage != 0 { + ret[n].Id = oidExtensionKeyUsage + ret[n].Critical = true + + var a [2]byte + a[0] = reverseBitsInAByte(byte(template.KeyUsage)) + a[1] = reverseBitsInAByte(byte(template.KeyUsage >> 8)) + + l := 1 + if a[1] != 0 { + l = 2 + } + + ret[n].Value, err = asn1.Marshal(asn1.BitString{Bytes: a[0:l], BitLength: l * 8}) + if err != nil { + return + } + n++ + } + + if template.BasicConstraintsValid { + ret[n].Id = oidExtensionBasicConstraints + ret[n].Value, err = asn1.Marshal(basicConstraints{template.IsCA, template.MaxPathLen}) + ret[n].Critical = true + if err != nil { + return + } + n++ + } + + if len(template.SubjectKeyId) > 0 { + ret[n].Id = oidExtensionSubjectKeyId + ret[n].Value, err = asn1.Marshal(template.SubjectKeyId) + if err != nil { + return + } + n++ + } + + if len(template.AuthorityKeyId) > 0 { + ret[n].Id = oidExtensionAuthorityKeyId + ret[n].Value, err = asn1.Marshal(authKeyId{template.AuthorityKeyId}) + if err != nil { + return + } + n++ + } + + if len(template.DNSNames) > 0 { + ret[n].Id = oidExtensionSubjectAltName + rawValues := make([]asn1.RawValue, len(template.DNSNames)) + for i, name := range template.DNSNames { + rawValues[i] = asn1.RawValue{Tag: 2, Class: 2, Bytes: []byte(name)} + } + ret[n].Value, err = asn1.Marshal(rawValues) + if err != nil { + return + } + n++ + } + + if len(template.PolicyIdentifiers) > 0 { + ret[n].Id = oidExtensionCertificatePolicies + policies := make([]policyInformation, len(template.PolicyIdentifiers)) + for i, policy := range template.PolicyIdentifiers { + policies[i].Policy = policy + } + ret[n].Value, err = asn1.Marshal(policies) + if err != nil { + return + } + n++ + } + + // Adding another extension here? Remember to update the maximum number + // of elements in the make() at the top of the function. + + return ret[0:n], nil +} + +var ( + oidSHA1WithRSA = []int{1, 2, 840, 113549, 1, 1, 5} + oidRSA = []int{1, 2, 840, 113549, 1, 1, 1} +) + +// CreateSelfSignedCertificate creates a new certificate based on +// a template. The following members of template are used: SerialNumber, +// Subject, NotBefore, NotAfter, KeyUsage, BasicConstraintsValid, IsCA, +// MaxPathLen, SubjectKeyId, DNSNames. +// +// The certificate is signed by parent. If parent is equal to template then the +// certificate is self-signed. The parameter pub is the public key of the +// signee and priv is the private key of the signer. +// +// The returned slice is the certificate in DER encoding. +func CreateCertificate(rand io.Reader, template, parent *Certificate, pub *rsa.PublicKey, priv *rsa.PrivateKey) (cert []byte, err os.Error) { + asn1PublicKey, err := asn1.Marshal(rsaPublicKey{ + N: asn1.RawValue{Tag: 2, Bytes: pub.N.Bytes()}, + E: pub.E, + }) + if err != nil { + return + } + + if len(parent.SubjectKeyId) > 0 { + template.AuthorityKeyId = parent.SubjectKeyId + } + + extensions, err := buildExtensions(template) + if err != nil { + return + } + + encodedPublicKey := asn1.BitString{BitLength: len(asn1PublicKey) * 8, Bytes: asn1PublicKey} + c := tbsCertificate{ + Version: 2, + SerialNumber: asn1.RawValue{Bytes: template.SerialNumber, Tag: 2}, + SignatureAlgorithm: algorithmIdentifier{oidSHA1WithRSA}, + Issuer: parent.Subject.toRDNSequence(), + Validity: validity{template.NotBefore, template.NotAfter}, + Subject: template.Subject.toRDNSequence(), + PublicKey: publicKeyInfo{algorithmIdentifier{oidRSA}, encodedPublicKey}, + Extensions: extensions, + } + + tbsCertContents, err := asn1.Marshal(c) + if err != nil { + return + } + + c.Raw = tbsCertContents + + h := sha1.New() + h.Write(tbsCertContents) + digest := h.Sum() + + signature, err := rsa.SignPKCS1v15(rand, priv, rsa.HashSHA1, digest) + if err != nil { + return + } + + cert, err = asn1.Marshal(certificate{ + c, + algorithmIdentifier{oidSHA1WithRSA}, + asn1.BitString{Bytes: signature, BitLength: len(signature) * 8}, + }) + return +} diff --git a/libgo/go/crypto/x509/x509_test.go b/libgo/go/crypto/x509/x509_test.go new file mode 100644 index 000000000..2fe47fdbe --- /dev/null +++ b/libgo/go/crypto/x509/x509_test.go @@ -0,0 +1,198 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package x509 + +import ( + "asn1" + "big" + "crypto/rand" + "crypto/rsa" + "encoding/hex" + "encoding/pem" + "reflect" + "testing" + "time" +) + +func TestParsePKCS1PrivateKey(t *testing.T) { + block, _ := pem.Decode([]byte(pemPrivateKey)) + priv, err := ParsePKCS1PrivateKey(block.Bytes) + if err != nil { + t.Errorf("Failed to parse private key: %s", err) + } + if !reflect.DeepEqual(priv, rsaPrivateKey) { + t.Errorf("got:%+v want:%+v", priv, rsaPrivateKey) + } +} + +var pemPrivateKey = `-----BEGIN RSA PRIVATE KEY----- +MIIBOgIBAAJBALKZD0nEffqM1ACuak0bijtqE2QrI/KLADv7l3kK3ppMyCuLKoF0 +fd7Ai2KW5ToIwzFofvJcS/STa6HA5gQenRUCAwEAAQJBAIq9amn00aS0h/CrjXqu +/ThglAXJmZhOMPVn4eiu7/ROixi9sex436MaVeMqSNf7Ex9a8fRNfWss7Sqd9eWu +RTUCIQDasvGASLqmjeffBNLTXV2A5g4t+kLVCpsEIZAycV5GswIhANEPLmax0ME/ +EO+ZJ79TJKN5yiGBRsv5yvx5UiHxajEXAiAhAol5N4EUyq6I9w1rYdhPMGpLfk7A +IU2snfRJ6Nq2CQIgFrPsWRCkV+gOYcajD17rEqmuLrdIRexpg8N1DOSXoJ8CIGlS +tAboUGBxTDq3ZroNism3DaMIbKPyYrAqhKov1h5V +-----END RSA PRIVATE KEY----- +` + +func bigFromString(s string) *big.Int { + ret := new(big.Int) + ret.SetString(s, 10) + return ret +} + +var rsaPrivateKey = &rsa.PrivateKey{ + PublicKey: rsa.PublicKey{ + N: bigFromString("9353930466774385905609975137998169297361893554149986716853295022578535724979677252958524466350471210367835187480748268864277464700638583474144061408845077"), + E: 65537, + }, + D: bigFromString("7266398431328116344057699379749222532279343923819063639497049039389899328538543087657733766554155839834519529439851673014800261285757759040931985506583861"), + P: bigFromString("98920366548084643601728869055592650835572950932266967461790948584315647051443"), + Q: bigFromString("94560208308847015747498523884063394671606671904944666360068158221458669711639"), +} + +type matchHostnamesTest struct { + pattern, host string + ok bool +} + +var matchHostnamesTests = []matchHostnamesTest{ + {"a.b.c", "a.b.c", true}, + {"a.b.c", "b.b.c", false}, + {"", "b.b.c", false}, + {"a.b.c", "", false}, + {"example.com", "example.com", true}, + {"example.com", "www.example.com", false}, + {"*.example.com", "www.example.com", true}, + {"*.example.com", "xyz.www.example.com", false}, + {"*.*.example.com", "xyz.www.example.com", true}, + {"*.www.*.com", "xyz.www.example.com", true}, +} + +func TestMatchHostnames(t *testing.T) { + for i, test := range matchHostnamesTests { + r := matchHostnames(test.pattern, test.host) + if r != test.ok { + t.Errorf("#%d mismatch got: %t want: %t", i, r, test.ok) + } + } +} + +func TestCertificateParse(t *testing.T) { + s, _ := hex.DecodeString(certBytes) + certs, err := ParseCertificates(s) + if err != nil { + t.Error(err) + } + if len(certs) != 2 { + t.Errorf("Wrong number of certs: got %d want 2", len(certs)) + return + } + + err = certs[0].CheckSignatureFrom(certs[1]) + if err != nil { + t.Error(err) + } + + if err := certs[0].VerifyHostname("mail.google.com"); err != nil { + t.Error(err) + } +} + +var certBytes = "308203223082028ba00302010202106edf0d9499fd4533dd1297fc42a93be1300d06092a864886" + + "f70d0101050500304c310b3009060355040613025a4131253023060355040a131c546861777465" + + "20436f6e73756c74696e67202850747929204c74642e311630140603550403130d546861777465" + + "20534743204341301e170d3039303332353136343932395a170d3130303332353136343932395a" + + "3069310b3009060355040613025553311330110603550408130a43616c69666f726e6961311630" + + "140603550407130d4d6f756e7461696e205669657731133011060355040a130a476f6f676c6520" + + "496e63311830160603550403130f6d61696c2e676f6f676c652e636f6d30819f300d06092a8648" + + "86f70d010101050003818d0030818902818100c5d6f892fccaf5614b064149e80a2c9581a218ef" + + "41ec35bd7a58125ae76f9ea54ddc893abbeb029f6b73616bf0ffd868791fba7af9c4aebf3706ba" + + "3eeaeed27435b4ddcfb157c05f351d66aa87fee0de072d66d773affbd36ab78bef090e0cc861a9" + + "03ac90dd98b51c9c41566c017f0beec3bff391051ffba0f5cc6850ad2a590203010001a381e730" + + "81e430280603551d250421301f06082b0601050507030106082b06010505070302060960864801" + + "86f842040130360603551d1f042f302d302ba029a0278625687474703a2f2f63726c2e74686177" + + "74652e636f6d2f54686177746553474343412e63726c307206082b060105050701010466306430" + + "2206082b060105050730018616687474703a2f2f6f6373702e7468617774652e636f6d303e0608" + + "2b060105050730028632687474703a2f2f7777772e7468617774652e636f6d2f7265706f736974" + + "6f72792f5468617774655f5347435f43412e637274300c0603551d130101ff04023000300d0609" + + "2a864886f70d01010505000381810062f1f3050ebc105e497c7aedf87e24d2f4a986bb3b837bd1" + + "9b91ebcad98b065992f6bd2b49b7d6d3cb2e427a99d606c7b1d46352527fac39e6a8b6726de5bf" + + "70212a52cba07634a5e332011bd1868e78eb5e3c93cf03072276786f207494feaa0ed9d53b2110" + + "a76571f90209cdae884385c882587030ee15f33d761e2e45a6bc308203233082028ca003020102" + + "020430000002300d06092a864886f70d0101050500305f310b3009060355040613025553311730" + + "15060355040a130e566572695369676e2c20496e632e31373035060355040b132e436c61737320" + + "33205075626c6963205072696d6172792043657274696669636174696f6e20417574686f726974" + + "79301e170d3034303531333030303030305a170d3134303531323233353935395a304c310b3009" + + "060355040613025a4131253023060355040a131c54686177746520436f6e73756c74696e672028" + + "50747929204c74642e311630140603550403130d5468617774652053474320434130819f300d06" + + "092a864886f70d010101050003818d0030818902818100d4d367d08d157faecd31fe7d1d91a13f" + + "0b713cacccc864fb63fc324b0794bd6f80ba2fe10493c033fc093323e90b742b71c403c6d2cde2" + + "2ff50963cdff48a500bfe0e7f388b72d32de9836e60aad007bc4644a3b847503f270927d0e62f5" + + "21ab693684317590f8bfc76c881b06957cc9e5a8de75a12c7a68dfd5ca1c875860190203010001" + + "a381fe3081fb30120603551d130101ff040830060101ff020100300b0603551d0f040403020106" + + "301106096086480186f842010104040302010630280603551d110421301fa41d301b3119301706" + + "035504031310507269766174654c6162656c332d313530310603551d1f042a30283026a024a022" + + "8620687474703a2f2f63726c2e766572697369676e2e636f6d2f706361332e63726c303206082b" + + "0601050507010104263024302206082b060105050730018616687474703a2f2f6f6373702e7468" + + "617774652e636f6d30340603551d25042d302b06082b0601050507030106082b06010505070302" + + "06096086480186f8420401060a6086480186f845010801300d06092a864886f70d010105050003" + + "81810055ac63eadea1ddd2905f9f0bce76be13518f93d9052bc81b774bad6950a1eededcfddb07" + + "e9e83994dcab72792f06bfab8170c4a8edea5334edef1e53d906c7562bd15cf4d18a8eb42bb137" + + "9048084225c53e8acb7feb6f04d16dc574a2f7a27c7b603c77cd0ece48027f012fb69b37e02a2a" + + "36dcd585d6ace53f546f961e05af" + +func TestCreateSelfSignedCertificate(t *testing.T) { + random := rand.Reader + + block, _ := pem.Decode([]byte(pemPrivateKey)) + priv, err := ParsePKCS1PrivateKey(block.Bytes) + if err != nil { + t.Errorf("Failed to parse private key: %s", err) + return + } + + template := Certificate{ + SerialNumber: []byte{1}, + Subject: Name{ + CommonName: "test.example.com", + Organization: []string{"Acme Co"}, + }, + NotBefore: time.SecondsToUTC(1000), + NotAfter: time.SecondsToUTC(100000), + + SubjectKeyId: []byte{1, 2, 3, 4}, + KeyUsage: KeyUsageCertSign, + + BasicConstraintsValid: true, + IsCA: true, + DNSNames: []string{"test.example.com"}, + + PolicyIdentifiers: []asn1.ObjectIdentifier{[]int{1, 2, 3}}, + } + + derBytes, err := CreateCertificate(random, &template, &template, &priv.PublicKey, priv) + if err != nil { + t.Errorf("Failed to create certificate: %s", err) + return + } + + cert, err := ParseCertificate(derBytes) + if err != nil { + t.Errorf("Failed to parse certificate: %s", err) + return + } + + if len(cert.PolicyIdentifiers) != 1 || !cert.PolicyIdentifiers[0].Equal(template.PolicyIdentifiers[0]) { + t.Errorf("Failed to parse policy identifiers: got:%#v want:%#v", cert.PolicyIdentifiers, template.PolicyIdentifiers) + } + + err = cert.CheckSignatureFrom(cert) + if err != nil { + t.Errorf("Signature verification failed: %s", err) + return + } +} diff --git a/libgo/go/crypto/xtea/block.go b/libgo/go/crypto/xtea/block.go new file mode 100644 index 000000000..3ac36d038 --- /dev/null +++ b/libgo/go/crypto/xtea/block.go @@ -0,0 +1,66 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + Implementation adapted from Needham and Wheeler's paper: + http://www.cix.co.uk/~klockstone/xtea.pdf + + A precalculated look up table is used during encryption/decryption for values that are based purely on the key. +*/ + +package xtea + +// XTEA is based on 64 rounds. +const numRounds = 64 + +// blockToUint32 reads an 8 byte slice into two uint32s. +// The block is treated as big endian. +func blockToUint32(src []byte) (uint32, uint32) { + r0 := uint32(src[0])<<24 | uint32(src[1])<<16 | uint32(src[2])<<8 | uint32(src[3]) + r1 := uint32(src[4])<<24 | uint32(src[5])<<16 | uint32(src[6])<<8 | uint32(src[7]) + return r0, r1 +} + +// uint32ToBlock writes two unint32s into an 8 byte data block. +// Values are written as big endian. +func uint32ToBlock(v0, v1 uint32, dst []byte) { + dst[0] = byte(v0 >> 24) + dst[1] = byte(v0 >> 16) + dst[2] = byte(v0 >> 8) + dst[3] = byte(v0) + dst[4] = byte(v1 >> 24) + dst[5] = byte(v1 >> 16) + dst[6] = byte(v1 >> 8) + dst[7] = byte(v1 >> 0) +} + +// encryptBlock encrypts a single 8 byte block using XTEA. +func encryptBlock(c *Cipher, dst, src []byte) { + v0, v1 := blockToUint32(src) + + // Two rounds of XTEA applied per loop + for i := 0; i < numRounds; { + v0 += ((v1<<4 ^ v1>>5) + v1) ^ c.table[i] + i++ + v1 += ((v0<<4 ^ v0>>5) + v0) ^ c.table[i] + i++ + } + + uint32ToBlock(v0, v1, dst) +} + +// decryptBlock decrypt a single 8 byte block using XTEA. +func decryptBlock(c *Cipher, dst, src []byte) { + v0, v1 := blockToUint32(src) + + // Two rounds of XTEA applied per loop + for i := numRounds; i > 0; { + i-- + v1 -= ((v0<<4 ^ v0>>5) + v0) ^ c.table[i] + i-- + v0 -= ((v1<<4 ^ v1>>5) + v1) ^ c.table[i] + } + + uint32ToBlock(v0, v1, dst) +} diff --git a/libgo/go/crypto/xtea/cipher.go b/libgo/go/crypto/xtea/cipher.go new file mode 100644 index 000000000..b0fa2a184 --- /dev/null +++ b/libgo/go/crypto/xtea/cipher.go @@ -0,0 +1,92 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This package implements XTEA encryption, as defined in Needham and +// Wheeler's 1997 technical report, "Tea extensions." +package xtea + +// For details, see http://www.cix.co.uk/~klockstone/xtea.pdf + +import ( + "os" + "strconv" +) + +// The XTEA block size in bytes. +const BlockSize = 8 + +// A Cipher is an instance of an XTEA cipher using a particular key. +// table contains a series of precalculated values that are used each round. +type Cipher struct { + table [64]uint32 +} + +type KeySizeError int + +func (k KeySizeError) String() string { + return "crypto/xtea: invalid key size " + strconv.Itoa(int(k)) +} + +// NewCipher creates and returns a new Cipher. +// The key argument should be the XTEA key. +// XTEA only supports 128 bit (16 byte) keys. +func NewCipher(key []byte) (*Cipher, os.Error) { + k := len(key) + switch k { + default: + return nil, KeySizeError(k) + case 16: + break + } + + c := new(Cipher) + initCipher(c, key) + + return c, nil +} + +// BlockSize returns the XTEA block size, 8 bytes. +// It is necessary to satisfy the Cipher interface in the +// package "crypto/block". +func (c *Cipher) BlockSize() int { return BlockSize } + +// Encrypt encrypts the 8 byte buffer src using the key and stores the result in dst. +// Note that for amounts of data larger than a block, +// it is not safe to just call Encrypt on successive blocks; +// instead, use an encryption mode like CBC (see crypto/block/cbc.go). +func (c *Cipher) Encrypt(dst, src []byte) { encryptBlock(c, dst, src) } + +// Decrypt decrypts the 8 byte buffer src using the key k and stores the result in dst. +func (c *Cipher) Decrypt(dst, src []byte) { decryptBlock(c, dst, src) } + +// Reset zeros the table, so that it will no longer appear in the process's memory. +func (c *Cipher) Reset() { + for i := 0; i < len(c.table); i++ { + c.table[i] = 0 + } +} + +// initCipher initializes the cipher context by creating a look up table +// of precalculated values that are based on the key. +func initCipher(c *Cipher, key []byte) { + // Load the key into four uint32s + var k [4]uint32 + for i := 0; i < len(k); i++ { + j := i << 2 // Multiply by 4 + k[i] = uint32(key[j+0])<<24 | uint32(key[j+1])<<16 | uint32(key[j+2])<<8 | uint32(key[j+3]) + } + + // Precalculate the table + const delta = 0x9E3779B9 + var sum uint32 = 0 + + // Two rounds of XTEA applied per loop + for i := 0; i < numRounds; { + c.table[i] = sum + k[sum&3] + i++ + sum += delta + c.table[i] = sum + k[(sum>>11)&3] + i++ + } +} diff --git a/libgo/go/crypto/xtea/xtea_test.go b/libgo/go/crypto/xtea/xtea_test.go new file mode 100644 index 000000000..03934f169 --- /dev/null +++ b/libgo/go/crypto/xtea/xtea_test.go @@ -0,0 +1,246 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package xtea + +import ( + "testing" +) + +// A sample test key for when we just want to initialise a cipher +var testKey = []byte{0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF} + +// Test that the block size for XTEA is correct +func TestBlocksize(t *testing.T) { + if BlockSize != 8 { + t.Errorf("BlockSize constant - expected 8, got %d", BlockSize) + return + } + + c, err := NewCipher(testKey) + if err != nil { + t.Errorf("NewCipher(%d bytes) = %s", len(testKey), err) + return + } + + result := c.BlockSize() + if result != 8 { + t.Errorf("BlockSize function - expected 8, gotr %d", result) + return + } +} + +// A series of test values to confirm that the Cipher.table array was initialised correctly +var testTable = []uint32{ + 0x00112233, 0x6B1568B8, 0xE28CE030, 0xC5089E2D, 0xC5089E2D, 0x1EFBD3A2, 0xA7845C2A, 0x78EF0917, + 0x78EF0917, 0x172682D0, 0x5B6AC714, 0x822AC955, 0x3DE68511, 0xDC1DFECA, 0x2062430E, 0x3611343F, + 0xF1CCEFFB, 0x900469B4, 0xD448ADF8, 0x2E3BE36D, 0xB6C46BF5, 0x994029F2, 0x994029F2, 0xF3335F67, + 0x6AAAD6DF, 0x4D2694DC, 0x4D2694DC, 0xEB5E0E95, 0x2FA252D9, 0x4551440A, 0x121E10D6, 0xB0558A8F, + 0xE388BDC3, 0x0A48C004, 0xC6047BC0, 0x643BF579, 0xA88039BD, 0x02736F32, 0x8AFBF7BA, 0x5C66A4A7, + 0x5C66A4A7, 0xC76AEB2C, 0x3EE262A4, 0x215E20A1, 0x215E20A1, 0x7B515616, 0x03D9DE9E, 0x1988CFCF, + 0xD5448B8B, 0x737C0544, 0xB7C04988, 0xDE804BC9, 0x9A3C0785, 0x3873813E, 0x7CB7C582, 0xD6AAFAF7, + 0x4E22726F, 0x309E306C, 0x309E306C, 0x8A9165E1, 0x1319EE69, 0xF595AC66, 0xF595AC66, 0x4F88E1DB, +} + +// Test that the cipher context is initialised correctly +func TestCipherInit(t *testing.T) { + c, err := NewCipher(testKey) + if err != nil { + t.Errorf("NewCipher(%d bytes) = %s", len(testKey), err) + return + } + + for i := 0; i < len(c.table); i++ { + if c.table[i] != testTable[i] { + t.Errorf("NewCipher() failed to initialise Cipher.table[%d] correctly. Expected %08X, got %08X", i, testTable[i], c.table[i]) + break + } + } +} + +// Test that invalid key sizes return an error +func TestInvalidKeySize(t *testing.T) { + // Test a long key + key := []byte{ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF, + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF, + } + + _, err := NewCipher(key) + if err == nil { + t.Errorf("Invalid key size %d didn't result in an error.", len(key)) + } + + // Test a short key + key = []byte{0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77} + + _, err = NewCipher(key) + if err == nil { + t.Errorf("Invalid key size %d didn't result in an error.", len(key)) + } +} + +// Test that we can correctly decode some bytes we have encoded +func TestEncodeDecode(t *testing.T) { + original := []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF} + input := original + output := make([]byte, BlockSize) + + c, err := NewCipher(testKey) + if err != nil { + t.Errorf("NewCipher(%d bytes) = %s", len(testKey), err) + return + } + + // Encrypt the input block + c.Encrypt(output, input) + + // Check that the output does not match the input + differs := false + for i := 0; i < len(input); i++ { + if output[i] != input[i] { + differs = true + break + } + } + if differs == false { + t.Error("Cipher.Encrypt: Failed to encrypt the input block.") + return + } + + // Decrypt the block we just encrypted + input = output + output = make([]byte, BlockSize) + c.Decrypt(output, input) + + // Check that the output from decrypt matches our initial input + for i := 0; i < len(input); i++ { + if output[i] != original[i] { + t.Errorf("Decrypted byte %d differed. Expected %02X, got %02X\n", i, original[i], output[i]) + return + } + } +} + +// Test Vectors +type CryptTest struct { + key []byte + plainText []byte + cipherText []byte +} + +var CryptTests = []CryptTest{ + // These were sourced from http://www.freemedialibrary.com/index.php/XTEA_test_vectors + { + []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f}, + []byte{0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48}, + []byte{0x49, 0x7d, 0xf3, 0xd0, 0x72, 0x61, 0x2c, 0xb5}, + }, + { + []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f}, + []byte{0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41}, + []byte{0xe7, 0x8f, 0x2d, 0x13, 0x74, 0x43, 0x41, 0xd8}, + }, + { + []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f}, + []byte{0x5a, 0x5b, 0x6e, 0x27, 0x89, 0x48, 0xd7, 0x7f}, + []byte{0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41}, + }, + { + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48}, + []byte{0xa0, 0x39, 0x05, 0x89, 0xf8, 0xb8, 0xef, 0xa5}, + }, + { + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41}, + []byte{0xed, 0x23, 0x37, 0x5a, 0x82, 0x1a, 0x8c, 0x2d}, + }, + { + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x70, 0xe1, 0x22, 0x5d, 0x6e, 0x4e, 0x76, 0x55}, + []byte{0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41}, + }, + + // These vectors are from http://wiki.secondlife.com/wiki/XTEA_Strong_Encryption_Implementation#Bouncy_Castle_C.23_API + { + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xDE, 0xE9, 0xD4, 0xD8, 0xF7, 0x13, 0x1E, 0xD9}, + }, + { + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08}, + []byte{0x06, 0x5C, 0x1B, 0x89, 0x75, 0xC6, 0xA8, 0x16}, + }, + { + []byte{0x01, 0x23, 0x45, 0x67, 0x12, 0x34, 0x56, 0x78, 0x23, 0x45, 0x67, 0x89, 0x34, 0x56, 0x78, 0x9A}, + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x1F, 0xF9, 0xA0, 0x26, 0x1A, 0xC6, 0x42, 0x64}, + }, + { + []byte{0x01, 0x23, 0x45, 0x67, 0x12, 0x34, 0x56, 0x78, 0x23, 0x45, 0x67, 0x89, 0x34, 0x56, 0x78, 0x9A}, + []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08}, + []byte{0x8C, 0x67, 0x15, 0x5B, 0x2E, 0xF9, 0x1E, 0xAD}, + }, +} + +// Test encryption +func TestCipherEncrypt(t *testing.T) { + for i, tt := range CryptTests { + c, err := NewCipher(tt.key) + if err != nil { + t.Errorf("NewCipher(%d bytes), vector %d = %s", len(tt.key), i, err) + continue + } + + out := make([]byte, len(tt.plainText)) + c.Encrypt(out, tt.plainText) + + for j := 0; j < len(out); j++ { + if out[j] != tt.cipherText[j] { + t.Errorf("Cipher.Encrypt %d: out[%d] = %02X, expected %02X", i, j, out[j], tt.cipherText[j]) + break + } + } + } +} + +// Test decryption +func TestCipherDecrypt(t *testing.T) { + for i, tt := range CryptTests { + c, err := NewCipher(tt.key) + if err != nil { + t.Errorf("NewCipher(%d bytes), vector %d = %s", len(tt.key), i, err) + continue + } + + out := make([]byte, len(tt.cipherText)) + c.Decrypt(out, tt.cipherText) + + for j := 0; j < len(out); j++ { + if out[j] != tt.plainText[j] { + t.Errorf("Cipher.Decrypt %d: out[%d] = %02X, expected %02X", i, j, out[j], tt.plainText[j]) + break + } + } + } +} + +// Test resetting the cipher context +func TestReset(t *testing.T) { + c, err := NewCipher(testKey) + if err != nil { + t.Errorf("NewCipher(%d bytes) = %s", len(testKey), err) + return + } + + c.Reset() + for i := 0; i < len(c.table); i++ { + if c.table[i] != 0 { + t.Errorf("Cipher.Reset: Failed to clear Cipher.table[%d]. expected 0, got %08X", i, c.table[i]) + return + } + } +} diff --git a/libgo/go/debug/dwarf/buf.go b/libgo/go/debug/dwarf/buf.go new file mode 100644 index 000000000..2d29cebdd --- /dev/null +++ b/libgo/go/debug/dwarf/buf.go @@ -0,0 +1,154 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Buffered reading and decoding of DWARF data streams. + +package dwarf + +import ( + "encoding/binary" + "os" + "strconv" +) + +// Data buffer being decoded. +type buf struct { + dwarf *Data + order binary.ByteOrder + name string + off Offset + data []byte + addrsize int + err os.Error +} + +func makeBuf(d *Data, name string, off Offset, data []byte, addrsize int) buf { + return buf{d, d.order, name, off, data, addrsize, nil} +} + +func (b *buf) uint8() uint8 { + if len(b.data) < 1 { + b.error("underflow") + return 0 + } + val := b.data[0] + b.data = b.data[1:] + b.off++ + return val +} + +func (b *buf) bytes(n int) []byte { + if len(b.data) < n { + b.error("underflow") + return nil + } + data := b.data[0:n] + b.data = b.data[n:] + b.off += Offset(n) + return data +} + +func (b *buf) skip(n int) { b.bytes(n) } + +func (b *buf) string() string { + for i := 0; i < len(b.data); i++ { + if b.data[i] == 0 { + s := string(b.data[0:i]) + b.data = b.data[i+1:] + b.off += Offset(i + 1) + return s + } + } + b.error("underflow") + return "" +} + +func (b *buf) uint16() uint16 { + a := b.bytes(2) + if a == nil { + return 0 + } + return b.order.Uint16(a) +} + +func (b *buf) uint32() uint32 { + a := b.bytes(4) + if a == nil { + return 0 + } + return b.order.Uint32(a) +} + +func (b *buf) uint64() uint64 { + a := b.bytes(8) + if a == nil { + return 0 + } + return b.order.Uint64(a) +} + +// Read a varint, which is 7 bits per byte, little endian. +// the 0x80 bit means read another byte. +func (b *buf) varint() (c uint64, bits uint) { + for i := 0; i < len(b.data); i++ { + byte := b.data[i] + c |= uint64(byte&0x7F) << bits + bits += 7 + if byte&0x80 == 0 { + b.off += Offset(i + 1) + b.data = b.data[i+1:] + return c, bits + } + } + return 0, 0 +} + +// Unsigned int is just a varint. +func (b *buf) uint() uint64 { + x, _ := b.varint() + return x +} + +// Signed int is a sign-extended varint. +func (b *buf) int() int64 { + ux, bits := b.varint() + x := int64(ux) + if x&(1<<(bits-1)) != 0 { + x |= -1 << bits + } + return x +} + +// Address-sized uint. +func (b *buf) addr() uint64 { + switch b.addrsize { + case 1: + return uint64(b.uint8()) + case 2: + return uint64(b.uint16()) + case 4: + return uint64(b.uint32()) + case 8: + return uint64(b.uint64()) + } + b.error("unknown address size") + return 0 +} + +func (b *buf) error(s string) { + if b.err == nil { + b.data = nil + b.err = DecodeError{b.name, b.off, s} + } +} + +type DecodeError struct { + Name string + Offset Offset + Error string +} + +func (e DecodeError) String() string { + return "decoding dwarf section " + e.Name + " at offset 0x" + strconv.Itob64(int64(e.Offset), 16) + ": " + e.Error +} diff --git a/libgo/go/debug/dwarf/const.go b/libgo/go/debug/dwarf/const.go new file mode 100644 index 000000000..1a3fec155 --- /dev/null +++ b/libgo/go/debug/dwarf/const.go @@ -0,0 +1,433 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Constants + +package dwarf + +import "strconv" + +// An Attr identifies the attribute type in a DWARF Entry's Field. +type Attr uint32 + +const ( + AttrSibling Attr = 0x01 + AttrLocation Attr = 0x02 + AttrName Attr = 0x03 + AttrOrdering Attr = 0x09 + AttrByteSize Attr = 0x0B + AttrBitOffset Attr = 0x0C + AttrBitSize Attr = 0x0D + AttrStmtList Attr = 0x10 + AttrLowpc Attr = 0x11 + AttrHighpc Attr = 0x12 + AttrLanguage Attr = 0x13 + AttrDiscr Attr = 0x15 + AttrDiscrValue Attr = 0x16 + AttrVisibility Attr = 0x17 + AttrImport Attr = 0x18 + AttrStringLength Attr = 0x19 + AttrCommonRef Attr = 0x1A + AttrCompDir Attr = 0x1B + AttrConstValue Attr = 0x1C + AttrContainingType Attr = 0x1D + AttrDefaultValue Attr = 0x1E + AttrInline Attr = 0x20 + AttrIsOptional Attr = 0x21 + AttrLowerBound Attr = 0x22 + AttrProducer Attr = 0x25 + AttrPrototyped Attr = 0x27 + AttrReturnAddr Attr = 0x2A + AttrStartScope Attr = 0x2C + AttrStrideSize Attr = 0x2E + AttrUpperBound Attr = 0x2F + AttrAbstractOrigin Attr = 0x31 + AttrAccessibility Attr = 0x32 + AttrAddrClass Attr = 0x33 + AttrArtificial Attr = 0x34 + AttrBaseTypes Attr = 0x35 + AttrCalling Attr = 0x36 + AttrCount Attr = 0x37 + AttrDataMemberLoc Attr = 0x38 + AttrDeclColumn Attr = 0x39 + AttrDeclFile Attr = 0x3A + AttrDeclLine Attr = 0x3B + AttrDeclaration Attr = 0x3C + AttrDiscrList Attr = 0x3D + AttrEncoding Attr = 0x3E + AttrExternal Attr = 0x3F + AttrFrameBase Attr = 0x40 + AttrFriend Attr = 0x41 + AttrIdentifierCase Attr = 0x42 + AttrMacroInfo Attr = 0x43 + AttrNamelistItem Attr = 0x44 + AttrPriority Attr = 0x45 + AttrSegment Attr = 0x46 + AttrSpecification Attr = 0x47 + AttrStaticLink Attr = 0x48 + AttrType Attr = 0x49 + AttrUseLocation Attr = 0x4A + AttrVarParam Attr = 0x4B + AttrVirtuality Attr = 0x4C + AttrVtableElemLoc Attr = 0x4D + AttrAllocated Attr = 0x4E + AttrAssociated Attr = 0x4F + AttrDataLocation Attr = 0x50 + AttrStride Attr = 0x51 + AttrEntrypc Attr = 0x52 + AttrUseUTF8 Attr = 0x53 + AttrExtension Attr = 0x54 + AttrRanges Attr = 0x55 + AttrTrampoline Attr = 0x56 + AttrCallColumn Attr = 0x57 + AttrCallFile Attr = 0x58 + AttrCallLine Attr = 0x59 + AttrDescription Attr = 0x5A +) + +var attrNames = [...]string{ + AttrSibling: "Sibling", + AttrLocation: "Location", + AttrName: "Name", + AttrOrdering: "Ordering", + AttrByteSize: "ByteSize", + AttrBitOffset: "BitOffset", + AttrBitSize: "BitSize", + AttrStmtList: "StmtList", + AttrLowpc: "Lowpc", + AttrHighpc: "Highpc", + AttrLanguage: "Language", + AttrDiscr: "Discr", + AttrDiscrValue: "DiscrValue", + AttrVisibility: "Visibility", + AttrImport: "Import", + AttrStringLength: "StringLength", + AttrCommonRef: "CommonRef", + AttrCompDir: "CompDir", + AttrConstValue: "ConstValue", + AttrContainingType: "ContainingType", + AttrDefaultValue: "DefaultValue", + AttrInline: "Inline", + AttrIsOptional: "IsOptional", + AttrLowerBound: "LowerBound", + AttrProducer: "Producer", + AttrPrototyped: "Prototyped", + AttrReturnAddr: "ReturnAddr", + AttrStartScope: "StartScope", + AttrStrideSize: "StrideSize", + AttrUpperBound: "UpperBound", + AttrAbstractOrigin: "AbstractOrigin", + AttrAccessibility: "Accessibility", + AttrAddrClass: "AddrClass", + AttrArtificial: "Artificial", + AttrBaseTypes: "BaseTypes", + AttrCalling: "Calling", + AttrCount: "Count", + AttrDataMemberLoc: "DataMemberLoc", + AttrDeclColumn: "DeclColumn", + AttrDeclFile: "DeclFile", + AttrDeclLine: "DeclLine", + AttrDeclaration: "Declaration", + AttrDiscrList: "DiscrList", + AttrEncoding: "Encoding", + AttrExternal: "External", + AttrFrameBase: "FrameBase", + AttrFriend: "Friend", + AttrIdentifierCase: "IdentifierCase", + AttrMacroInfo: "MacroInfo", + AttrNamelistItem: "NamelistItem", + AttrPriority: "Priority", + AttrSegment: "Segment", + AttrSpecification: "Specification", + AttrStaticLink: "StaticLink", + AttrType: "Type", + AttrUseLocation: "UseLocation", + AttrVarParam: "VarParam", + AttrVirtuality: "Virtuality", + AttrVtableElemLoc: "VtableElemLoc", + AttrAllocated: "Allocated", + AttrAssociated: "Associated", + AttrDataLocation: "DataLocation", + AttrStride: "Stride", + AttrEntrypc: "Entrypc", + AttrUseUTF8: "UseUTF8", + AttrExtension: "Extension", + AttrRanges: "Ranges", + AttrTrampoline: "Trampoline", + AttrCallColumn: "CallColumn", + AttrCallFile: "CallFile", + AttrCallLine: "CallLine", + AttrDescription: "Description", +} + +func (a Attr) String() string { + if int(a) < len(attrNames) { + s := attrNames[a] + if s != "" { + return s + } + } + return strconv.Itoa(int(a)) +} + +func (a Attr) GoString() string { + if int(a) < len(attrNames) { + s := attrNames[a] + if s != "" { + return "dwarf.Attr" + s + } + } + return "dwarf.Attr(" + strconv.Itoa64(int64(a)) + ")" +} + +// A format is a DWARF data encoding format. +type format uint32 + +const ( + // value formats + formAddr format = 0x01 + formDwarfBlock2 format = 0x03 + formDwarfBlock4 format = 0x04 + formData2 format = 0x05 + formData4 format = 0x06 + formData8 format = 0x07 + formString format = 0x08 + formDwarfBlock format = 0x09 + formDwarfBlock1 format = 0x0A + formData1 format = 0x0B + formFlag format = 0x0C + formSdata format = 0x0D + formStrp format = 0x0E + formUdata format = 0x0F + formRefAddr format = 0x10 + formRef1 format = 0x11 + formRef2 format = 0x12 + formRef4 format = 0x13 + formRef8 format = 0x14 + formRefUdata format = 0x15 + formIndirect format = 0x16 +) + +// A Tag is the classification (the type) of an Entry. +type Tag uint32 + +const ( + TagArrayType Tag = 0x01 + TagClassType Tag = 0x02 + TagEntryPoint Tag = 0x03 + TagEnumerationType Tag = 0x04 + TagFormalParameter Tag = 0x05 + TagImportedDeclaration Tag = 0x08 + TagLabel Tag = 0x0A + TagLexDwarfBlock Tag = 0x0B + TagMember Tag = 0x0D + TagPointerType Tag = 0x0F + TagReferenceType Tag = 0x10 + TagCompileUnit Tag = 0x11 + TagStringType Tag = 0x12 + TagStructType Tag = 0x13 + TagSubroutineType Tag = 0x15 + TagTypedef Tag = 0x16 + TagUnionType Tag = 0x17 + TagUnspecifiedParameters Tag = 0x18 + TagVariant Tag = 0x19 + TagCommonDwarfBlock Tag = 0x1A + TagCommonInclusion Tag = 0x1B + TagInheritance Tag = 0x1C + TagInlinedSubroutine Tag = 0x1D + TagModule Tag = 0x1E + TagPtrToMemberType Tag = 0x1F + TagSetType Tag = 0x20 + TagSubrangeType Tag = 0x21 + TagWithStmt Tag = 0x22 + TagAccessDeclaration Tag = 0x23 + TagBaseType Tag = 0x24 + TagCatchDwarfBlock Tag = 0x25 + TagConstType Tag = 0x26 + TagConstant Tag = 0x27 + TagEnumerator Tag = 0x28 + TagFileType Tag = 0x29 + TagFriend Tag = 0x2A + TagNamelist Tag = 0x2B + TagNamelistItem Tag = 0x2C + TagPackedType Tag = 0x2D + TagSubprogram Tag = 0x2E + TagTemplateTypeParameter Tag = 0x2F + TagTemplateValueParameter Tag = 0x30 + TagThrownType Tag = 0x31 + TagTryDwarfBlock Tag = 0x32 + TagVariantPart Tag = 0x33 + TagVariable Tag = 0x34 + TagVolatileType Tag = 0x35 + TagDwarfProcedure Tag = 0x36 + TagRestrictType Tag = 0x37 + TagInterfaceType Tag = 0x38 + TagNamespace Tag = 0x39 + TagImportedModule Tag = 0x3A + TagUnspecifiedType Tag = 0x3B + TagPartialUnit Tag = 0x3C + TagImportedUnit Tag = 0x3D + TagMutableType Tag = 0x3E +) + +var tagNames = [...]string{ + TagArrayType: "ArrayType", + TagClassType: "ClassType", + TagEntryPoint: "EntryPoint", + TagEnumerationType: "EnumerationType", + TagFormalParameter: "FormalParameter", + TagImportedDeclaration: "ImportedDeclaration", + TagLabel: "Label", + TagLexDwarfBlock: "LexDwarfBlock", + TagMember: "Member", + TagPointerType: "PointerType", + TagReferenceType: "ReferenceType", + TagCompileUnit: "CompileUnit", + TagStringType: "StringType", + TagStructType: "StructType", + TagSubroutineType: "SubroutineType", + TagTypedef: "Typedef", + TagUnionType: "UnionType", + TagUnspecifiedParameters: "UnspecifiedParameters", + TagVariant: "Variant", + TagCommonDwarfBlock: "CommonDwarfBlock", + TagCommonInclusion: "CommonInclusion", + TagInheritance: "Inheritance", + TagInlinedSubroutine: "InlinedSubroutine", + TagModule: "Module", + TagPtrToMemberType: "PtrToMemberType", + TagSetType: "SetType", + TagSubrangeType: "SubrangeType", + TagWithStmt: "WithStmt", + TagAccessDeclaration: "AccessDeclaration", + TagBaseType: "BaseType", + TagCatchDwarfBlock: "CatchDwarfBlock", + TagConstType: "ConstType", + TagConstant: "Constant", + TagEnumerator: "Enumerator", + TagFileType: "FileType", + TagFriend: "Friend", + TagNamelist: "Namelist", + TagNamelistItem: "NamelistItem", + TagPackedType: "PackedType", + TagSubprogram: "Subprogram", + TagTemplateTypeParameter: "TemplateTypeParameter", + TagTemplateValueParameter: "TemplateValueParameter", + TagThrownType: "ThrownType", + TagTryDwarfBlock: "TryDwarfBlock", + TagVariantPart: "VariantPart", + TagVariable: "Variable", + TagVolatileType: "VolatileType", + TagDwarfProcedure: "DwarfProcedure", + TagRestrictType: "RestrictType", + TagInterfaceType: "InterfaceType", + TagNamespace: "Namespace", + TagImportedModule: "ImportedModule", + TagUnspecifiedType: "UnspecifiedType", + TagPartialUnit: "PartialUnit", + TagImportedUnit: "ImportedUnit", + TagMutableType: "MutableType", +} + +func (t Tag) String() string { + if int(t) < len(tagNames) { + s := tagNames[t] + if s != "" { + return s + } + } + return strconv.Itoa(int(t)) +} + +func (t Tag) GoString() string { + if int(t) < len(tagNames) { + s := tagNames[t] + if s != "" { + return "dwarf.Tag" + s + } + } + return "dwarf.Tag(" + strconv.Itoa64(int64(t)) + ")" +} + +// Location expression operators. +// The debug info encodes value locations like 8(R3) +// as a sequence of these op codes. +// This package does not implement full expressions; +// the opPlusUconst operator is expected by the type parser. +const ( + opAddr = 0x03 /* 1 op, const addr */ + opDeref = 0x06 + opConst1u = 0x08 /* 1 op, 1 byte const */ + opConst1s = 0x09 /* " signed */ + opConst2u = 0x0A /* 1 op, 2 byte const */ + opConst2s = 0x0B /* " signed */ + opConst4u = 0x0C /* 1 op, 4 byte const */ + opConst4s = 0x0D /* " signed */ + opConst8u = 0x0E /* 1 op, 8 byte const */ + opConst8s = 0x0F /* " signed */ + opConstu = 0x10 /* 1 op, LEB128 const */ + opConsts = 0x11 /* " signed */ + opDup = 0x12 + opDrop = 0x13 + opOver = 0x14 + opPick = 0x15 /* 1 op, 1 byte stack index */ + opSwap = 0x16 + opRot = 0x17 + opXderef = 0x18 + opAbs = 0x19 + opAnd = 0x1A + opDiv = 0x1B + opMinus = 0x1C + opMod = 0x1D + opMul = 0x1E + opNeg = 0x1F + opNot = 0x20 + opOr = 0x21 + opPlus = 0x22 + opPlusUconst = 0x23 /* 1 op, ULEB128 addend */ + opShl = 0x24 + opShr = 0x25 + opShra = 0x26 + opXor = 0x27 + opSkip = 0x2F /* 1 op, signed 2-byte constant */ + opBra = 0x28 /* 1 op, signed 2-byte constant */ + opEq = 0x29 + opGe = 0x2A + opGt = 0x2B + opLe = 0x2C + opLt = 0x2D + opNe = 0x2E + opLit0 = 0x30 + /* OpLitN = OpLit0 + N for N = 0..31 */ + opReg0 = 0x50 + /* OpRegN = OpReg0 + N for N = 0..31 */ + opBreg0 = 0x70 /* 1 op, signed LEB128 constant */ + /* OpBregN = OpBreg0 + N for N = 0..31 */ + opRegx = 0x90 /* 1 op, ULEB128 register */ + opFbreg = 0x91 /* 1 op, SLEB128 offset */ + opBregx = 0x92 /* 2 op, ULEB128 reg; SLEB128 off */ + opPiece = 0x93 /* 1 op, ULEB128 size of piece */ + opDerefSize = 0x94 /* 1-byte size of data retrieved */ + opXderefSize = 0x95 /* 1-byte size of data retrieved */ + opNop = 0x96 + /* next four new in Dwarf v3 */ + opPushObjAddr = 0x97 + opCall2 = 0x98 /* 2-byte offset of DIE */ + opCall4 = 0x99 /* 4-byte offset of DIE */ + opCallRef = 0x9A /* 4- or 8- byte offset of DIE */ + /* 0xE0-0xFF reserved for user-specific */ +) + +// Basic type encodings -- the value for AttrEncoding in a TagBaseType Entry. +const ( + encAddress = 0x01 + encBoolean = 0x02 + encComplexFloat = 0x03 + encFloat = 0x04 + encSigned = 0x05 + encSignedChar = 0x06 + encUnsigned = 0x07 + encUnsignedChar = 0x08 + encImaginaryFloat = 0x09 +) diff --git a/libgo/go/debug/dwarf/entry.go b/libgo/go/debug/dwarf/entry.go new file mode 100644 index 000000000..549e5c2cc --- /dev/null +++ b/libgo/go/debug/dwarf/entry.go @@ -0,0 +1,343 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// DWARF debug information entry parser. +// An entry is a sequence of data items of a given format. +// The first word in the entry is an index into what DWARF +// calls the ``abbreviation table.'' An abbreviation is really +// just a type descriptor: it's an array of attribute tag/value format pairs. + +package dwarf + +import "os" + +// a single entry's description: a sequence of attributes +type abbrev struct { + tag Tag + children bool + field []afield +} + +type afield struct { + attr Attr + fmt format +} + +// a map from entry format ids to their descriptions +type abbrevTable map[uint32]abbrev + +// ParseAbbrev returns the abbreviation table that starts at byte off +// in the .debug_abbrev section. +func (d *Data) parseAbbrev(off uint32) (abbrevTable, os.Error) { + if m, ok := d.abbrevCache[off]; ok { + return m, nil + } + + data := d.abbrev + if off > uint32(len(data)) { + data = nil + } else { + data = data[off:] + } + b := makeBuf(d, "abbrev", 0, data, 0) + + // Error handling is simplified by the buf getters + // returning an endless stream of 0s after an error. + m := make(abbrevTable) + for { + // Table ends with id == 0. + id := uint32(b.uint()) + if id == 0 { + break + } + + // Walk over attributes, counting. + n := 0 + b1 := b // Read from copy of b. + b1.uint() + b1.uint8() + for { + tag := b1.uint() + fmt := b1.uint() + if tag == 0 && fmt == 0 { + break + } + n++ + } + if b1.err != nil { + return nil, b1.err + } + + // Walk over attributes again, this time writing them down. + var a abbrev + a.tag = Tag(b.uint()) + a.children = b.uint8() != 0 + a.field = make([]afield, n) + for i := range a.field { + a.field[i].attr = Attr(b.uint()) + a.field[i].fmt = format(b.uint()) + } + b.uint() + b.uint() + + m[id] = a + } + if b.err != nil { + return nil, b.err + } + d.abbrevCache[off] = m + return m, nil +} + +// An entry is a sequence of attribute/value pairs. +type Entry struct { + Offset Offset // offset of Entry in DWARF info + Tag Tag // tag (kind of Entry) + Children bool // whether Entry is followed by children + Field []Field +} + +// A Field is a single attribute/value pair in an Entry. +type Field struct { + Attr Attr + Val interface{} +} + +// Val returns the value associated with attribute Attr in Entry, +// or nil if there is no such attribute. +// +// A common idiom is to merge the check for nil return with +// the check that the value has the expected dynamic type, as in: +// v, ok := e.Val(AttrSibling).(int64); +// +func (e *Entry) Val(a Attr) interface{} { + for _, f := range e.Field { + if f.Attr == a { + return f.Val + } + } + return nil +} + +// An Offset represents the location of an Entry within the DWARF info. +// (See Reader.Seek.) +type Offset uint32 + +// Entry reads a single entry from buf, decoding +// according to the given abbreviation table. +func (b *buf) entry(atab abbrevTable, ubase Offset) *Entry { + off := b.off + id := uint32(b.uint()) + if id == 0 { + return &Entry{} + } + a, ok := atab[id] + if !ok { + b.error("unknown abbreviation table index") + return nil + } + e := &Entry{ + Offset: off, + Tag: a.tag, + Children: a.children, + Field: make([]Field, len(a.field)), + } + for i := range e.Field { + e.Field[i].Attr = a.field[i].attr + fmt := a.field[i].fmt + if fmt == formIndirect { + fmt = format(b.uint()) + } + var val interface{} + switch fmt { + default: + b.error("unknown entry attr format") + + // address + case formAddr: + val = b.addr() + + // block + case formDwarfBlock1: + val = b.bytes(int(b.uint8())) + case formDwarfBlock2: + val = b.bytes(int(b.uint16())) + case formDwarfBlock4: + val = b.bytes(int(b.uint32())) + case formDwarfBlock: + val = b.bytes(int(b.uint())) + + // constant + case formData1: + val = int64(b.uint8()) + case formData2: + val = int64(b.uint16()) + case formData4: + val = int64(b.uint32()) + case formData8: + val = int64(b.uint64()) + case formSdata: + val = int64(b.int()) + case formUdata: + val = int64(b.uint()) + + // flag + case formFlag: + val = b.uint8() == 1 + + // reference to other entry + case formRefAddr: + val = Offset(b.addr()) + case formRef1: + val = Offset(b.uint8()) + ubase + case formRef2: + val = Offset(b.uint16()) + ubase + case formRef4: + val = Offset(b.uint32()) + ubase + case formRef8: + val = Offset(b.uint64()) + ubase + case formRefUdata: + val = Offset(b.uint()) + ubase + + // string + case formString: + val = b.string() + case formStrp: + off := b.uint32() // offset into .debug_str + if b.err != nil { + return nil + } + b1 := makeBuf(b.dwarf, "str", 0, b.dwarf.str, 0) + b1.skip(int(off)) + val = b1.string() + if b1.err != nil { + b.err = b1.err + return nil + } + } + e.Field[i].Val = val + } + if b.err != nil { + return nil + } + return e +} + +// A Reader allows reading Entry structures from a DWARF ``info'' section. +// The Entry structures are arranged in a tree. The Reader's Next function +// return successive entries from a pre-order traversal of the tree. +// If an entry has children, its Children field will be true, and the children +// follow, terminated by an Entry with Tag 0. +type Reader struct { + b buf + d *Data + err os.Error + unit int + lastChildren bool // .Children of last entry returned by Next + lastSibling Offset // .Val(AttrSibling) of last entry returned by Next +} + +// Reader returns a new Reader for Data. +// The reader is positioned at byte offset 0 in the DWARF ``info'' section. +func (d *Data) Reader() *Reader { + r := &Reader{d: d} + r.Seek(0) + return r +} + +// Seek positions the Reader at offset off in the encoded entry stream. +// Offset 0 can be used to denote the first entry. +func (r *Reader) Seek(off Offset) { + d := r.d + r.err = nil + r.lastChildren = false + if off == 0 { + if len(d.unit) == 0 { + return + } + u := &d.unit[0] + r.unit = 0 + r.b = makeBuf(r.d, "info", u.off, u.data, u.addrsize) + return + } + + // TODO(rsc): binary search (maybe a new package) + var i int + var u *unit + for i = range d.unit { + u = &d.unit[i] + if u.off <= off && off < u.off+Offset(len(u.data)) { + r.unit = i + r.b = makeBuf(r.d, "info", off, u.data[off-u.off:], u.addrsize) + return + } + } + r.err = os.NewError("offset out of range") +} + +// maybeNextUnit advances to the next unit if this one is finished. +func (r *Reader) maybeNextUnit() { + for len(r.b.data) == 0 && r.unit+1 < len(r.d.unit) { + r.unit++ + u := &r.d.unit[r.unit] + r.b = makeBuf(r.d, "info", u.off, u.data, u.addrsize) + } +} + +// Next reads the next entry from the encoded entry stream. +// It returns nil, nil when it reaches the end of the section. +// It returns an error if the current offset is invalid or the data at the +// offset cannot be decoded as a valid Entry. +func (r *Reader) Next() (*Entry, os.Error) { + if r.err != nil { + return nil, r.err + } + r.maybeNextUnit() + if len(r.b.data) == 0 { + return nil, nil + } + u := &r.d.unit[r.unit] + e := r.b.entry(u.atable, u.base) + if r.b.err != nil { + r.err = r.b.err + return nil, r.err + } + if e != nil { + r.lastChildren = e.Children + if r.lastChildren { + r.lastSibling, _ = e.Val(AttrSibling).(Offset) + } + } else { + r.lastChildren = false + } + return e, nil +} + +// SkipChildren skips over the child entries associated with +// the last Entry returned by Next. If that Entry did not have +// children or Next has not been called, SkipChildren is a no-op. +func (r *Reader) SkipChildren() { + if r.err != nil || !r.lastChildren { + return + } + + // If the last entry had a sibling attribute, + // that attribute gives the offset of the next + // sibling, so we can avoid decoding the + // child subtrees. + if r.lastSibling >= r.b.off { + r.Seek(r.lastSibling) + return + } + + for { + e, err := r.Next() + if err != nil || e == nil || e.Tag == 0 { + break + } + if e.Children { + r.SkipChildren() + } + } +} diff --git a/libgo/go/debug/dwarf/open.go b/libgo/go/debug/dwarf/open.go new file mode 100644 index 000000000..cb009e0e0 --- /dev/null +++ b/libgo/go/debug/dwarf/open.go @@ -0,0 +1,80 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This package provides access to DWARF debugging information +// loaded from executable files, as defined in the DWARF 2.0 Standard +// at http://dwarfstd.org/doc/dwarf-2.0.0.pdf +package dwarf + +import ( + "encoding/binary" + "os" +) + +// Data represents the DWARF debugging information +// loaded from an executable file (for example, an ELF or Mach-O executable). +type Data struct { + // raw data + abbrev []byte + aranges []byte + frame []byte + info []byte + line []byte + pubnames []byte + ranges []byte + str []byte + + // parsed data + abbrevCache map[uint32]abbrevTable + addrsize int + order binary.ByteOrder + typeCache map[Offset]Type + unit []unit +} + +// New returns a new Data object initialized from the given parameters. +// Clients should typically use [TODO(rsc): method to be named later] instead of calling +// New directly. +// +// The []byte arguments are the data from the corresponding debug section +// in the object file; for example, for an ELF object, abbrev is the contents of +// the ".debug_abbrev" section. +func New(abbrev, aranges, frame, info, line, pubnames, ranges, str []byte) (*Data, os.Error) { + d := &Data{ + abbrev: abbrev, + aranges: aranges, + frame: frame, + info: info, + line: line, + pubnames: pubnames, + ranges: ranges, + str: str, + abbrevCache: make(map[uint32]abbrevTable), + typeCache: make(map[Offset]Type), + } + + // Sniff .debug_info to figure out byte order. + // bytes 4:6 are the version, a tiny 16-bit number (1, 2, 3). + if len(d.info) < 6 { + return nil, DecodeError{"info", Offset(len(d.info)), "too short"} + } + x, y := d.info[4], d.info[5] + switch { + case x == 0 && y == 0: + return nil, DecodeError{"info", 4, "unsupported version 0"} + case x == 0: + d.order = binary.BigEndian + case y == 0: + d.order = binary.LittleEndian + default: + return nil, DecodeError{"info", 4, "cannot determine byte order"} + } + + u, err := d.parseUnits() + if err != nil { + return nil, err + } + d.unit = u + return d, nil +} diff --git a/libgo/go/debug/dwarf/testdata/typedef.c b/libgo/go/debug/dwarf/testdata/typedef.c new file mode 100644 index 000000000..664d021ce --- /dev/null +++ b/libgo/go/debug/dwarf/testdata/typedef.c @@ -0,0 +1,79 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +Linux ELF: +gcc -gdwarf-2 -m64 -c typedef.c && gcc -gdwarf-2 -m64 -o typedef.elf typedef.o + +OS X Mach-O: +gcc -gdwarf-2 -m64 -c typedef.c -o typedef.macho +*/ +#include + +typedef volatile int* t_ptr_volatile_int; +typedef const char *t_ptr_const_char; +typedef long t_long; +typedef unsigned short t_ushort; +typedef int t_func_int_of_float_double(float, double); +typedef int (*t_ptr_func_int_of_float_double)(float, double); +typedef int (*t_ptr_func_int_of_float_complex)(float complex); +typedef int (*t_ptr_func_int_of_double_complex)(double complex); +typedef int (*t_ptr_func_int_of_long_double_complex)(long double complex); +typedef int *t_func_ptr_int_of_char_schar_uchar(char, signed char, unsigned char); +typedef void t_func_void_of_char(char); +typedef void t_func_void_of_void(void); +typedef void t_func_void_of_ptr_char_dots(char*, ...); +typedef struct my_struct { + volatile int vi; + char x : 1; + int y : 4; + long long array[40]; +} t_my_struct; +typedef union my_union { + volatile int vi; + char x : 1; + int y : 4; + long long array[40]; +} t_my_union; +typedef enum my_enum { + e1 = 1, + e2 = 2, + e3 = -5, + e4 = 1000000000000000LL, +} t_my_enum; + +typedef struct list t_my_list; +struct list { + short val; + t_my_list *next; +}; + +typedef struct tree { + struct tree *left, *right; + unsigned long long val; +} t_my_tree; + +t_ptr_volatile_int *a2; +t_ptr_const_char **a3a; +t_long *a4; +t_ushort *a5; +t_func_int_of_float_double *a6; +t_ptr_func_int_of_float_double *a7; +t_func_ptr_int_of_char_schar_uchar *a8; +t_func_void_of_char *a9; +t_func_void_of_void *a10; +t_func_void_of_ptr_char_dots *a11; +t_my_struct *a12; +t_my_union *a12a; +t_my_enum *a13; +t_my_list *a14; +t_my_tree *a15; +t_ptr_func_int_of_float_complex *a16; +t_ptr_func_int_of_double_complex *a17; +t_ptr_func_int_of_long_double_complex *a18; + +int main() +{ + return 0; +} diff --git a/libgo/go/debug/dwarf/testdata/typedef.elf b/libgo/go/debug/dwarf/testdata/typedef.elf new file mode 100755 index 000000000..44df8da9b Binary files /dev/null and b/libgo/go/debug/dwarf/testdata/typedef.elf differ diff --git a/libgo/go/debug/dwarf/testdata/typedef.macho b/libgo/go/debug/dwarf/testdata/typedef.macho new file mode 100644 index 000000000..41019c1e1 Binary files /dev/null and b/libgo/go/debug/dwarf/testdata/typedef.macho differ diff --git a/libgo/go/debug/dwarf/type.go b/libgo/go/debug/dwarf/type.go new file mode 100644 index 000000000..902a545f8 --- /dev/null +++ b/libgo/go/debug/dwarf/type.go @@ -0,0 +1,583 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// DWARF type information structures. +// The format is heavily biased toward C, but for simplicity +// the String methods use a pseudo-Go syntax. + +package dwarf + +import ( + "os" + "strconv" +) + +// A Type conventionally represents a pointer to any of the +// specific Type structures (CharType, StructType, etc.). +type Type interface { + Common() *CommonType + String() string + Size() int64 +} + +// A CommonType holds fields common to multiple types. +// If a field is not known or not applicable for a given type, +// the zero value is used. +type CommonType struct { + ByteSize int64 // size of value of this type, in bytes + Name string // name that can be used to refer to type +} + +func (c *CommonType) Common() *CommonType { return c } + +func (c *CommonType) Size() int64 { return c.ByteSize } + +// Basic types + +// A BasicType holds fields common to all basic types. +type BasicType struct { + CommonType + BitSize int64 + BitOffset int64 +} + +func (b *BasicType) Basic() *BasicType { return b } + +func (t *BasicType) String() string { + if t.Name != "" { + return t.Name + } + return "?" +} + +// A CharType represents a signed character type. +type CharType struct { + BasicType +} + +// A UcharType represents an unsigned character type. +type UcharType struct { + BasicType +} + +// An IntType represents a signed integer type. +type IntType struct { + BasicType +} + +// A UintType represents an unsigned integer type. +type UintType struct { + BasicType +} + +// A FloatType represents a floating point type. +type FloatType struct { + BasicType +} + +// A ComplexType represents a complex floating point type. +type ComplexType struct { + BasicType +} + +// A BoolType represents a boolean type. +type BoolType struct { + BasicType +} + +// An AddrType represents a machine address type. +type AddrType struct { + BasicType +} + +// qualifiers + +// A QualType represents a type that has the C/C++ "const", "restrict", or "volatile" qualifier. +type QualType struct { + CommonType + Qual string + Type Type +} + +func (t *QualType) String() string { return t.Qual + " " + t.Type.String() } + +func (t *QualType) Size() int64 { return t.Type.Size() } + +// An ArrayType represents a fixed size array type. +type ArrayType struct { + CommonType + Type Type + StrideBitSize int64 // if > 0, number of bits to hold each element + Count int64 // if == -1, an incomplete array, like char x[]. +} + +func (t *ArrayType) String() string { + return "[" + strconv.Itoa64(t.Count) + "]" + t.Type.String() +} + +func (t *ArrayType) Size() int64 { return t.Count * t.Type.Size() } + +// A VoidType represents the C void type. +type VoidType struct { + CommonType +} + +func (t *VoidType) String() string { return "void" } + +// A PtrType represents a pointer type. +type PtrType struct { + CommonType + Type Type +} + +func (t *PtrType) String() string { return "*" + t.Type.String() } + +// A StructType represents a struct, union, or C++ class type. +type StructType struct { + CommonType + StructName string + Kind string // "struct", "union", or "class". + Field []*StructField + Incomplete bool // if true, struct, union, class is declared but not defined +} + +// A StructField represents a field in a struct, union, or C++ class type. +type StructField struct { + Name string + Type Type + ByteOffset int64 + ByteSize int64 + BitOffset int64 // within the ByteSize bytes at ByteOffset + BitSize int64 // zero if not a bit field +} + +func (t *StructType) String() string { + if t.StructName != "" { + return t.Kind + " " + t.StructName + } + return t.Defn() +} + +func (t *StructType) Defn() string { + s := t.Kind + if t.StructName != "" { + s += " " + t.StructName + } + if t.Incomplete { + s += " /*incomplete*/" + return s + } + s += " {" + for i, f := range t.Field { + if i > 0 { + s += "; " + } + s += f.Name + " " + f.Type.String() + s += "@" + strconv.Itoa64(f.ByteOffset) + if f.BitSize > 0 { + s += " : " + strconv.Itoa64(f.BitSize) + s += "@" + strconv.Itoa64(f.BitOffset) + } + } + s += "}" + return s +} + +// An EnumType represents an enumerated type. +// The only indication of its native integer type is its ByteSize +// (inside CommonType). +type EnumType struct { + CommonType + EnumName string + Val []*EnumValue +} + +// An EnumValue represents a single enumeration value. +type EnumValue struct { + Name string + Val int64 +} + +func (t *EnumType) String() string { + s := "enum" + if t.EnumName != "" { + s += " " + t.EnumName + } + s += " {" + for i, v := range t.Val { + if i > 0 { + s += "; " + } + s += v.Name + "=" + strconv.Itoa64(v.Val) + } + s += "}" + return s +} + +// A FuncType represents a function type. +type FuncType struct { + CommonType + ReturnType Type + ParamType []Type +} + +func (t *FuncType) String() string { + s := "func(" + for i, t := range t.ParamType { + if i > 0 { + s += ", " + } + s += t.String() + } + s += ")" + if t.ReturnType != nil { + s += " " + t.ReturnType.String() + } + return s +} + +// A DotDotDotType represents the variadic ... function parameter. +type DotDotDotType struct { + CommonType +} + +func (t *DotDotDotType) String() string { return "..." } + +// A TypedefType represents a named type. +type TypedefType struct { + CommonType + Type Type +} + +func (t *TypedefType) String() string { return t.Name } + +func (t *TypedefType) Size() int64 { return t.Type.Size() } + +func (d *Data) Type(off Offset) (Type, os.Error) { + if t, ok := d.typeCache[off]; ok { + return t, nil + } + + r := d.Reader() + r.Seek(off) + e, err := r.Next() + if err != nil { + return nil, err + } + if e == nil || e.Offset != off { + return nil, DecodeError{"info", off, "no type at offset"} + } + + // Parse type from Entry. + // Must always set d.typeCache[off] before calling + // d.Type recursively, to handle circular types correctly. + var typ Type + + // Get next child; set err if error happens. + next := func() *Entry { + if !e.Children { + return nil + } + kid, err1 := r.Next() + if err1 != nil { + err = err1 + return nil + } + if kid == nil { + err = DecodeError{"info", r.b.off, "unexpected end of DWARF entries"} + return nil + } + if kid.Tag == 0 { + return nil + } + return kid + } + + // Get Type referred to by Entry's AttrType field. + // Set err if error happens. Not having a type is an error. + typeOf := func(e *Entry) Type { + toff, ok := e.Val(AttrType).(Offset) + if !ok { + // It appears that no Type means "void". + return new(VoidType) + } + var t Type + if t, err = d.Type(toff); err != nil { + return nil + } + return t + } + + switch e.Tag { + case TagArrayType: + // Multi-dimensional array. (DWARF v2 §5.4) + // Attributes: + // AttrType:subtype [required] + // AttrStrideSize: size in bits of each element of the array + // AttrByteSize: size of entire array + // Children: + // TagSubrangeType or TagEnumerationType giving one dimension. + // dimensions are in left to right order. + t := new(ArrayType) + typ = t + d.typeCache[off] = t + if t.Type = typeOf(e); err != nil { + goto Error + } + t.StrideBitSize, _ = e.Val(AttrStrideSize).(int64) + + // Accumulate dimensions, + ndim := 0 + for kid := next(); kid != nil; kid = next() { + // TODO(rsc): Can also be TagEnumerationType + // but haven't seen that in the wild yet. + switch kid.Tag { + case TagSubrangeType: + max, ok := kid.Val(AttrUpperBound).(int64) + if !ok { + max = -2 // Count == -1, as in x[]. + } + if ndim == 0 { + t.Count = max + 1 + } else { + // Multidimensional array. + // Create new array type underneath this one. + t.Type = &ArrayType{Type: t.Type, Count: max + 1} + } + ndim++ + case TagEnumerationType: + err = DecodeError{"info", kid.Offset, "cannot handle enumeration type as array bound"} + goto Error + } + } + if ndim == 0 { + err = DecodeError{"info", e.Offset, "missing dimension for array"} + goto Error + } + + case TagBaseType: + // Basic type. (DWARF v2 §5.1) + // Attributes: + // AttrName: name of base type in programming language of the compilation unit [required] + // AttrEncoding: encoding value for type (encFloat etc) [required] + // AttrByteSize: size of type in bytes [required] + // AttrBitOffset: for sub-byte types, size in bits + // AttrBitSize: for sub-byte types, bit offset of high order bit in the AttrByteSize bytes + name, _ := e.Val(AttrName).(string) + enc, ok := e.Val(AttrEncoding).(int64) + if !ok { + err = DecodeError{"info", e.Offset, "missing encoding attribute for " + name} + goto Error + } + switch enc { + default: + err = DecodeError{"info", e.Offset, "unrecognized encoding attribute value"} + goto Error + + case encAddress: + typ = new(AddrType) + case encBoolean: + typ = new(BoolType) + case encComplexFloat: + typ = new(ComplexType) + case encFloat: + typ = new(FloatType) + case encSigned: + typ = new(IntType) + case encUnsigned: + typ = new(UintType) + case encSignedChar: + typ = new(CharType) + case encUnsignedChar: + typ = new(UcharType) + } + d.typeCache[off] = typ + t := typ.(interface { + Basic() *BasicType + }).Basic() + t.Name = name + t.BitSize, _ = e.Val(AttrBitSize).(int64) + t.BitOffset, _ = e.Val(AttrBitOffset).(int64) + + case TagClassType, TagStructType, TagUnionType: + // Structure, union, or class type. (DWARF v2 §5.5) + // Attributes: + // AttrName: name of struct, union, or class + // AttrByteSize: byte size [required] + // AttrDeclaration: if true, struct/union/class is incomplete + // Children: + // TagMember to describe one member. + // AttrName: name of member [required] + // AttrType: type of member [required] + // AttrByteSize: size in bytes + // AttrBitOffset: bit offset within bytes for bit fields + // AttrBitSize: bit size for bit fields + // AttrDataMemberLoc: location within struct [required for struct, class] + // There is much more to handle C++, all ignored for now. + t := new(StructType) + typ = t + d.typeCache[off] = t + switch e.Tag { + case TagClassType: + t.Kind = "class" + case TagStructType: + t.Kind = "struct" + case TagUnionType: + t.Kind = "union" + } + t.StructName, _ = e.Val(AttrName).(string) + t.Incomplete = e.Val(AttrDeclaration) != nil + t.Field = make([]*StructField, 0, 8) + for kid := next(); kid != nil; kid = next() { + if kid.Tag == TagMember { + f := new(StructField) + if f.Type = typeOf(kid); err != nil { + goto Error + } + if loc, ok := kid.Val(AttrDataMemberLoc).([]byte); ok { + b := makeBuf(d, "location", 0, loc, d.addrsize) + if b.uint8() != opPlusUconst { + err = DecodeError{"info", kid.Offset, "unexpected opcode"} + goto Error + } + f.ByteOffset = int64(b.uint()) + if b.err != nil { + err = b.err + goto Error + } + } + f.Name, _ = kid.Val(AttrName).(string) + f.ByteSize, _ = kid.Val(AttrByteSize).(int64) + f.BitOffset, _ = kid.Val(AttrBitOffset).(int64) + f.BitSize, _ = kid.Val(AttrBitSize).(int64) + t.Field = append(t.Field, f) + } + } + + case TagConstType, TagVolatileType, TagRestrictType: + // Type modifier (DWARF v2 §5.2) + // Attributes: + // AttrType: subtype + t := new(QualType) + typ = t + d.typeCache[off] = t + if t.Type = typeOf(e); err != nil { + goto Error + } + switch e.Tag { + case TagConstType: + t.Qual = "const" + case TagRestrictType: + t.Qual = "restrict" + case TagVolatileType: + t.Qual = "volatile" + } + + case TagEnumerationType: + // Enumeration type (DWARF v2 §5.6) + // Attributes: + // AttrName: enum name if any + // AttrByteSize: bytes required to represent largest value + // Children: + // TagEnumerator: + // AttrName: name of constant + // AttrConstValue: value of constant + t := new(EnumType) + typ = t + d.typeCache[off] = t + t.EnumName, _ = e.Val(AttrName).(string) + t.Val = make([]*EnumValue, 0, 8) + for kid := next(); kid != nil; kid = next() { + if kid.Tag == TagEnumerator { + f := new(EnumValue) + f.Name, _ = kid.Val(AttrName).(string) + f.Val, _ = kid.Val(AttrConstValue).(int64) + n := len(t.Val) + if n >= cap(t.Val) { + val := make([]*EnumValue, n, n*2) + copy(val, t.Val) + t.Val = val + } + t.Val = t.Val[0 : n+1] + t.Val[n] = f + } + } + + case TagPointerType: + // Type modifier (DWARF v2 §5.2) + // Attributes: + // AttrType: subtype [not required! void* has no AttrType] + // AttrAddrClass: address class [ignored] + t := new(PtrType) + typ = t + d.typeCache[off] = t + if e.Val(AttrType) == nil { + t.Type = &VoidType{} + break + } + t.Type = typeOf(e) + + case TagSubroutineType: + // Subroutine type. (DWARF v2 §5.7) + // Attributes: + // AttrType: type of return value if any + // AttrName: possible name of type [ignored] + // AttrPrototyped: whether used ANSI C prototye [ignored] + // Children: + // TagFormalParameter: typed parameter + // AttrType: type of parameter + // TagUnspecifiedParameter: final ... + t := new(FuncType) + typ = t + d.typeCache[off] = t + if t.ReturnType = typeOf(e); err != nil { + goto Error + } + t.ParamType = make([]Type, 0, 8) + for kid := next(); kid != nil; kid = next() { + var tkid Type + switch kid.Tag { + default: + continue + case TagFormalParameter: + if tkid = typeOf(kid); err != nil { + goto Error + } + case TagUnspecifiedParameters: + tkid = &DotDotDotType{} + } + t.ParamType = append(t.ParamType, tkid) + } + + case TagTypedef: + // Typedef (DWARF v2 §5.3) + // Attributes: + // AttrName: name [required] + // AttrType: type definition [required] + t := new(TypedefType) + typ = t + d.typeCache[off] = t + t.Name, _ = e.Val(AttrName).(string) + t.Type = typeOf(e) + } + + if err != nil { + goto Error + } + + b, ok := e.Val(AttrByteSize).(int64) + if !ok { + b = -1 + } + typ.Common().ByteSize = b + + return typ, nil + +Error: + // If the parse fails, take the type out of the cache + // so that the next call with this offset doesn't hit + // the cache and return success. + d.typeCache[off] = nil, false + return nil, err +} diff --git a/libgo/go/debug/dwarf/type_test.go b/libgo/go/debug/dwarf/type_test.go new file mode 100644 index 000000000..e01f7353a --- /dev/null +++ b/libgo/go/debug/dwarf/type_test.go @@ -0,0 +1,112 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package dwarf_test + +import ( + . "debug/dwarf" + "debug/elf" + "debug/macho" + "testing" +) + +var typedefTests = map[string]string{ + "t_ptr_volatile_int": "*volatile int", + "t_ptr_const_char": "*const char", + "t_long": "long int", + "t_ushort": "short unsigned int", + "t_func_int_of_float_double": "func(float, double) int", + "t_ptr_func_int_of_float_double": "*func(float, double) int", + "t_ptr_func_int_of_float_complex": "*func(complex float) int", + "t_ptr_func_int_of_double_complex": "*func(complex double) int", + "t_ptr_func_int_of_long_double_complex": "*func(complex long double) int", + "t_func_ptr_int_of_char_schar_uchar": "func(char, signed char, unsigned char) *int", + "t_func_void_of_char": "func(char) void", + "t_func_void_of_void": "func() void", + "t_func_void_of_ptr_char_dots": "func(*char, ...) void", + "t_my_struct": "struct my_struct {vi volatile int@0; x char@4 : 1@7; y int@4 : 4@27; array [40]long long int@8}", + "t_my_union": "union my_union {vi volatile int@0; x char@0 : 1@7; y int@0 : 4@28; array [40]long long int@0}", + "t_my_enum": "enum my_enum {e1=1; e2=2; e3=-5; e4=1000000000000000}", + "t_my_list": "struct list {val short int@0; next *t_my_list@8}", + "t_my_tree": "struct tree {left *struct tree@0; right *struct tree@8; val long long unsigned int@16}", +} + +func elfData(t *testing.T, name string) *Data { + f, err := elf.Open(name) + if err != nil { + t.Fatal(err) + } + + d, err := f.DWARF() + if err != nil { + t.Fatal(err) + } + return d +} + +func machoData(t *testing.T, name string) *Data { + f, err := macho.Open(name) + if err != nil { + t.Fatal(err) + } + + d, err := f.DWARF() + if err != nil { + t.Fatal(err) + } + return d +} + + +func TestTypedefsELF(t *testing.T) { testTypedefs(t, elfData(t, "testdata/typedef.elf")) } + +func TestTypedefsMachO(t *testing.T) { + testTypedefs(t, machoData(t, "testdata/typedef.macho")) +} + +func testTypedefs(t *testing.T, d *Data) { + r := d.Reader() + seen := make(map[string]bool) + for { + e, err := r.Next() + if err != nil { + t.Fatal("r.Next:", err) + } + if e == nil { + break + } + if e.Tag == TagTypedef { + typ, err := d.Type(e.Offset) + if err != nil { + t.Fatal("d.Type:", err) + } + t1 := typ.(*TypedefType) + var typstr string + if ts, ok := t1.Type.(*StructType); ok { + typstr = ts.Defn() + } else { + typstr = t1.Type.String() + } + + if want, ok := typedefTests[t1.Name]; ok { + if seen[t1.Name] { + t.Errorf("multiple definitions for %s", t1.Name) + } + seen[t1.Name] = true + if typstr != want { + t.Errorf("%s:\n\thave %s\n\twant %s", t1.Name, typstr, want) + } + } + } + if e.Tag != TagCompileUnit { + r.SkipChildren() + } + } + + for k := range typedefTests { + if !seen[k] { + t.Errorf("missing %s", k) + } + } +} diff --git a/libgo/go/debug/dwarf/unit.go b/libgo/go/debug/dwarf/unit.go new file mode 100644 index 000000000..02cb363b4 --- /dev/null +++ b/libgo/go/debug/dwarf/unit.go @@ -0,0 +1,62 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package dwarf + +import ( + "os" + "strconv" +) + +// DWARF debug info is split into a sequence of compilation units. +// Each unit has its own abbreviation table and address size. + +type unit struct { + base Offset // byte offset of header within the aggregate info + off Offset // byte offset of data within the aggregate info + data []byte + atable abbrevTable + addrsize int +} + +func (d *Data) parseUnits() ([]unit, os.Error) { + // Count units. + nunit := 0 + b := makeBuf(d, "info", 0, d.info, 0) + for len(b.data) > 0 { + b.skip(int(b.uint32())) + nunit++ + } + if b.err != nil { + return nil, b.err + } + + // Again, this time writing them down. + b = makeBuf(d, "info", 0, d.info, 0) + units := make([]unit, nunit) + for i := range units { + u := &units[i] + u.base = b.off + n := b.uint32() + if vers := b.uint16(); vers != 2 { + b.error("unsupported DWARF version " + strconv.Itoa(int(vers))) + break + } + atable, err := d.parseAbbrev(b.uint32()) + if err != nil { + if b.err == nil { + b.err = err + } + break + } + u.atable = atable + u.addrsize = int(b.uint8()) + u.off = b.off + u.data = b.bytes(int(n - (2 + 4 + 1))) + } + if b.err != nil { + return nil, b.err + } + return units, nil +} diff --git a/libgo/go/debug/elf/elf.go b/libgo/go/debug/elf/elf.go new file mode 100644 index 000000000..74e979986 --- /dev/null +++ b/libgo/go/debug/elf/elf.go @@ -0,0 +1,1506 @@ +/* + * ELF constants and data structures + * + * Derived from: + * $FreeBSD: src/sys/sys/elf32.h,v 1.8.14.1 2005/12/30 22:13:58 marcel Exp $ + * $FreeBSD: src/sys/sys/elf64.h,v 1.10.14.1 2005/12/30 22:13:58 marcel Exp $ + * $FreeBSD: src/sys/sys/elf_common.h,v 1.15.8.1 2005/12/30 22:13:58 marcel Exp $ + * $FreeBSD: src/sys/alpha/include/elf.h,v 1.14 2003/09/25 01:10:22 peter Exp $ + * $FreeBSD: src/sys/amd64/include/elf.h,v 1.18 2004/08/03 08:21:48 dfr Exp $ + * $FreeBSD: src/sys/arm/include/elf.h,v 1.5.2.1 2006/06/30 21:42:52 cognet Exp $ + * $FreeBSD: src/sys/i386/include/elf.h,v 1.16 2004/08/02 19:12:17 dfr Exp $ + * $FreeBSD: src/sys/powerpc/include/elf.h,v 1.7 2004/11/02 09:47:01 ssouhlal Exp $ + * $FreeBSD: src/sys/sparc64/include/elf.h,v 1.12 2003/09/25 01:10:26 peter Exp $ + * + * Copyright (c) 1996-1998 John D. Polstra. All rights reserved. + * Copyright (c) 2001 David E. O'Brien + * Portions Copyright 2009 The Go Authors. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +package elf + +import "strconv" + +/* + * Constants + */ + +// Indexes into the Header.Ident array. +const ( + EI_CLASS = 4 /* Class of machine. */ + EI_DATA = 5 /* Data format. */ + EI_VERSION = 6 /* ELF format version. */ + EI_OSABI = 7 /* Operating system / ABI identification */ + EI_ABIVERSION = 8 /* ABI version */ + EI_PAD = 9 /* Start of padding (per SVR4 ABI). */ + EI_NIDENT = 16 /* Size of e_ident array. */ +) + +// Initial magic number for ELF files. +const ELFMAG = "\177ELF" + +// Version is found in Header.Ident[EI_VERSION] and Header.Version. +type Version byte + +const ( + EV_NONE Version = 0 + EV_CURRENT Version = 1 +) + +var versionStrings = []intName{ + {0, "EV_NONE"}, + {1, "EV_CURRENT"}, +} + +func (i Version) String() string { return stringName(uint32(i), versionStrings, false) } +func (i Version) GoString() string { return stringName(uint32(i), versionStrings, true) } + +// Class is found in Header.Ident[EI_CLASS] and Header.Class. +type Class byte + +const ( + ELFCLASSNONE Class = 0 /* Unknown class. */ + ELFCLASS32 Class = 1 /* 32-bit architecture. */ + ELFCLASS64 Class = 2 /* 64-bit architecture. */ +) + +var classStrings = []intName{ + {0, "ELFCLASSNONE"}, + {1, "ELFCLASS32"}, + {2, "ELFCLASS64"}, +} + +func (i Class) String() string { return stringName(uint32(i), classStrings, false) } +func (i Class) GoString() string { return stringName(uint32(i), classStrings, true) } + +// Data is found in Header.Ident[EI_DATA] and Header.Data. +type Data byte + +const ( + ELFDATANONE Data = 0 /* Unknown data format. */ + ELFDATA2LSB Data = 1 /* 2's complement little-endian. */ + ELFDATA2MSB Data = 2 /* 2's complement big-endian. */ +) + +var dataStrings = []intName{ + {0, "ELFDATANONE"}, + {1, "ELFDATA2LSB"}, + {2, "ELFDATA2MSB"}, +} + +func (i Data) String() string { return stringName(uint32(i), dataStrings, false) } +func (i Data) GoString() string { return stringName(uint32(i), dataStrings, true) } + +// OSABI is found in Header.Ident[EI_OSABI] and Header.OSABI. +type OSABI byte + +const ( + ELFOSABI_NONE OSABI = 0 /* UNIX System V ABI */ + ELFOSABI_HPUX OSABI = 1 /* HP-UX operating system */ + ELFOSABI_NETBSD OSABI = 2 /* NetBSD */ + ELFOSABI_LINUX OSABI = 3 /* GNU/Linux */ + ELFOSABI_HURD OSABI = 4 /* GNU/Hurd */ + ELFOSABI_86OPEN OSABI = 5 /* 86Open common IA32 ABI */ + ELFOSABI_SOLARIS OSABI = 6 /* Solaris */ + ELFOSABI_AIX OSABI = 7 /* AIX */ + ELFOSABI_IRIX OSABI = 8 /* IRIX */ + ELFOSABI_FREEBSD OSABI = 9 /* FreeBSD */ + ELFOSABI_TRU64 OSABI = 10 /* TRU64 UNIX */ + ELFOSABI_MODESTO OSABI = 11 /* Novell Modesto */ + ELFOSABI_OPENBSD OSABI = 12 /* OpenBSD */ + ELFOSABI_OPENVMS OSABI = 13 /* Open VMS */ + ELFOSABI_NSK OSABI = 14 /* HP Non-Stop Kernel */ + ELFOSABI_ARM OSABI = 97 /* ARM */ + ELFOSABI_STANDALONE OSABI = 255 /* Standalone (embedded) application */ +) + +var osabiStrings = []intName{ + {0, "ELFOSABI_NONE"}, + {1, "ELFOSABI_HPUX"}, + {2, "ELFOSABI_NETBSD"}, + {3, "ELFOSABI_LINUX"}, + {4, "ELFOSABI_HURD"}, + {5, "ELFOSABI_86OPEN"}, + {6, "ELFOSABI_SOLARIS"}, + {7, "ELFOSABI_AIX"}, + {8, "ELFOSABI_IRIX"}, + {9, "ELFOSABI_FREEBSD"}, + {10, "ELFOSABI_TRU64"}, + {11, "ELFOSABI_MODESTO"}, + {12, "ELFOSABI_OPENBSD"}, + {13, "ELFOSABI_OPENVMS"}, + {14, "ELFOSABI_NSK"}, + {97, "ELFOSABI_ARM"}, + {255, "ELFOSABI_STANDALONE"}, +} + +func (i OSABI) String() string { return stringName(uint32(i), osabiStrings, false) } +func (i OSABI) GoString() string { return stringName(uint32(i), osabiStrings, true) } + +// Type is found in Header.Type. +type Type uint16 + +const ( + ET_NONE Type = 0 /* Unknown type. */ + ET_REL Type = 1 /* Relocatable. */ + ET_EXEC Type = 2 /* Executable. */ + ET_DYN Type = 3 /* Shared object. */ + ET_CORE Type = 4 /* Core file. */ + ET_LOOS Type = 0xfe00 /* First operating system specific. */ + ET_HIOS Type = 0xfeff /* Last operating system-specific. */ + ET_LOPROC Type = 0xff00 /* First processor-specific. */ + ET_HIPROC Type = 0xffff /* Last processor-specific. */ +) + +var typeStrings = []intName{ + {0, "ET_NONE"}, + {1, "ET_REL"}, + {2, "ET_EXEC"}, + {3, "ET_DYN"}, + {4, "ET_CORE"}, + {0xfe00, "ET_LOOS"}, + {0xfeff, "ET_HIOS"}, + {0xff00, "ET_LOPROC"}, + {0xffff, "ET_HIPROC"}, +} + +func (i Type) String() string { return stringName(uint32(i), typeStrings, false) } +func (i Type) GoString() string { return stringName(uint32(i), typeStrings, true) } + +// Machine is found in Header.Machine. +type Machine uint16 + +const ( + EM_NONE Machine = 0 /* Unknown machine. */ + EM_M32 Machine = 1 /* AT&T WE32100. */ + EM_SPARC Machine = 2 /* Sun SPARC. */ + EM_386 Machine = 3 /* Intel i386. */ + EM_68K Machine = 4 /* Motorola 68000. */ + EM_88K Machine = 5 /* Motorola 88000. */ + EM_860 Machine = 7 /* Intel i860. */ + EM_MIPS Machine = 8 /* MIPS R3000 Big-Endian only. */ + EM_S370 Machine = 9 /* IBM System/370. */ + EM_MIPS_RS3_LE Machine = 10 /* MIPS R3000 Little-Endian. */ + EM_PARISC Machine = 15 /* HP PA-RISC. */ + EM_VPP500 Machine = 17 /* Fujitsu VPP500. */ + EM_SPARC32PLUS Machine = 18 /* SPARC v8plus. */ + EM_960 Machine = 19 /* Intel 80960. */ + EM_PPC Machine = 20 /* PowerPC 32-bit. */ + EM_PPC64 Machine = 21 /* PowerPC 64-bit. */ + EM_S390 Machine = 22 /* IBM System/390. */ + EM_V800 Machine = 36 /* NEC V800. */ + EM_FR20 Machine = 37 /* Fujitsu FR20. */ + EM_RH32 Machine = 38 /* TRW RH-32. */ + EM_RCE Machine = 39 /* Motorola RCE. */ + EM_ARM Machine = 40 /* ARM. */ + EM_SH Machine = 42 /* Hitachi SH. */ + EM_SPARCV9 Machine = 43 /* SPARC v9 64-bit. */ + EM_TRICORE Machine = 44 /* Siemens TriCore embedded processor. */ + EM_ARC Machine = 45 /* Argonaut RISC Core. */ + EM_H8_300 Machine = 46 /* Hitachi H8/300. */ + EM_H8_300H Machine = 47 /* Hitachi H8/300H. */ + EM_H8S Machine = 48 /* Hitachi H8S. */ + EM_H8_500 Machine = 49 /* Hitachi H8/500. */ + EM_IA_64 Machine = 50 /* Intel IA-64 Processor. */ + EM_MIPS_X Machine = 51 /* Stanford MIPS-X. */ + EM_COLDFIRE Machine = 52 /* Motorola ColdFire. */ + EM_68HC12 Machine = 53 /* Motorola M68HC12. */ + EM_MMA Machine = 54 /* Fujitsu MMA. */ + EM_PCP Machine = 55 /* Siemens PCP. */ + EM_NCPU Machine = 56 /* Sony nCPU. */ + EM_NDR1 Machine = 57 /* Denso NDR1 microprocessor. */ + EM_STARCORE Machine = 58 /* Motorola Star*Core processor. */ + EM_ME16 Machine = 59 /* Toyota ME16 processor. */ + EM_ST100 Machine = 60 /* STMicroelectronics ST100 processor. */ + EM_TINYJ Machine = 61 /* Advanced Logic Corp. TinyJ processor. */ + EM_X86_64 Machine = 62 /* Advanced Micro Devices x86-64 */ + + /* Non-standard or deprecated. */ + EM_486 Machine = 6 /* Intel i486. */ + EM_MIPS_RS4_BE Machine = 10 /* MIPS R4000 Big-Endian */ + EM_ALPHA_STD Machine = 41 /* Digital Alpha (standard value). */ + EM_ALPHA Machine = 0x9026 /* Alpha (written in the absence of an ABI) */ +) + +var machineStrings = []intName{ + {0, "EM_NONE"}, + {1, "EM_M32"}, + {2, "EM_SPARC"}, + {3, "EM_386"}, + {4, "EM_68K"}, + {5, "EM_88K"}, + {7, "EM_860"}, + {8, "EM_MIPS"}, + {9, "EM_S370"}, + {10, "EM_MIPS_RS3_LE"}, + {15, "EM_PARISC"}, + {17, "EM_VPP500"}, + {18, "EM_SPARC32PLUS"}, + {19, "EM_960"}, + {20, "EM_PPC"}, + {21, "EM_PPC64"}, + {22, "EM_S390"}, + {36, "EM_V800"}, + {37, "EM_FR20"}, + {38, "EM_RH32"}, + {39, "EM_RCE"}, + {40, "EM_ARM"}, + {42, "EM_SH"}, + {43, "EM_SPARCV9"}, + {44, "EM_TRICORE"}, + {45, "EM_ARC"}, + {46, "EM_H8_300"}, + {47, "EM_H8_300H"}, + {48, "EM_H8S"}, + {49, "EM_H8_500"}, + {50, "EM_IA_64"}, + {51, "EM_MIPS_X"}, + {52, "EM_COLDFIRE"}, + {53, "EM_68HC12"}, + {54, "EM_MMA"}, + {55, "EM_PCP"}, + {56, "EM_NCPU"}, + {57, "EM_NDR1"}, + {58, "EM_STARCORE"}, + {59, "EM_ME16"}, + {60, "EM_ST100"}, + {61, "EM_TINYJ"}, + {62, "EM_X86_64"}, + + /* Non-standard or deprecated. */ + {6, "EM_486"}, + {10, "EM_MIPS_RS4_BE"}, + {41, "EM_ALPHA_STD"}, + {0x9026, "EM_ALPHA"}, +} + +func (i Machine) String() string { return stringName(uint32(i), machineStrings, false) } +func (i Machine) GoString() string { return stringName(uint32(i), machineStrings, true) } + +// Special section indices. +type SectionIndex int + +const ( + SHN_UNDEF SectionIndex = 0 /* Undefined, missing, irrelevant. */ + SHN_LORESERVE SectionIndex = 0xff00 /* First of reserved range. */ + SHN_LOPROC SectionIndex = 0xff00 /* First processor-specific. */ + SHN_HIPROC SectionIndex = 0xff1f /* Last processor-specific. */ + SHN_LOOS SectionIndex = 0xff20 /* First operating system-specific. */ + SHN_HIOS SectionIndex = 0xff3f /* Last operating system-specific. */ + SHN_ABS SectionIndex = 0xfff1 /* Absolute values. */ + SHN_COMMON SectionIndex = 0xfff2 /* Common data. */ + SHN_XINDEX SectionIndex = 0xffff /* Escape -- index stored elsewhere. */ + SHN_HIRESERVE SectionIndex = 0xffff /* Last of reserved range. */ +) + +var shnStrings = []intName{ + {0, "SHN_UNDEF"}, + {0xff00, "SHN_LOPROC"}, + {0xff20, "SHN_LOOS"}, + {0xfff1, "SHN_ABS"}, + {0xfff2, "SHN_COMMON"}, + {0xffff, "SHN_XINDEX"}, +} + +func (i SectionIndex) String() string { return stringName(uint32(i), shnStrings, false) } +func (i SectionIndex) GoString() string { return stringName(uint32(i), shnStrings, true) } + +// Section type. +type SectionType uint32 + +const ( + SHT_NULL SectionType = 0 /* inactive */ + SHT_PROGBITS SectionType = 1 /* program defined information */ + SHT_SYMTAB SectionType = 2 /* symbol table section */ + SHT_STRTAB SectionType = 3 /* string table section */ + SHT_RELA SectionType = 4 /* relocation section with addends */ + SHT_HASH SectionType = 5 /* symbol hash table section */ + SHT_DYNAMIC SectionType = 6 /* dynamic section */ + SHT_NOTE SectionType = 7 /* note section */ + SHT_NOBITS SectionType = 8 /* no space section */ + SHT_REL SectionType = 9 /* relocation section - no addends */ + SHT_SHLIB SectionType = 10 /* reserved - purpose unknown */ + SHT_DYNSYM SectionType = 11 /* dynamic symbol table section */ + SHT_INIT_ARRAY SectionType = 14 /* Initialization function pointers. */ + SHT_FINI_ARRAY SectionType = 15 /* Termination function pointers. */ + SHT_PREINIT_ARRAY SectionType = 16 /* Pre-initialization function ptrs. */ + SHT_GROUP SectionType = 17 /* Section group. */ + SHT_SYMTAB_SHNDX SectionType = 18 /* Section indexes (see SHN_XINDEX). */ + SHT_LOOS SectionType = 0x60000000 /* First of OS specific semantics */ + SHT_HIOS SectionType = 0x6fffffff /* Last of OS specific semantics */ + SHT_LOPROC SectionType = 0x70000000 /* reserved range for processor */ + SHT_HIPROC SectionType = 0x7fffffff /* specific section header types */ + SHT_LOUSER SectionType = 0x80000000 /* reserved range for application */ + SHT_HIUSER SectionType = 0xffffffff /* specific indexes */ +) + +var shtStrings = []intName{ + {0, "SHT_NULL"}, + {1, "SHT_PROGBITS"}, + {2, "SHT_SYMTAB"}, + {3, "SHT_STRTAB"}, + {4, "SHT_RELA"}, + {5, "SHT_HASH"}, + {6, "SHT_DYNAMIC"}, + {7, "SHT_NOTE"}, + {8, "SHT_NOBITS"}, + {9, "SHT_REL"}, + {10, "SHT_SHLIB"}, + {11, "SHT_DYNSYM"}, + {14, "SHT_INIT_ARRAY"}, + {15, "SHT_FINI_ARRAY"}, + {16, "SHT_PREINIT_ARRAY"}, + {17, "SHT_GROUP"}, + {18, "SHT_SYMTAB_SHNDX"}, + {0x60000000, "SHT_LOOS"}, + {0x6fffffff, "SHT_HIOS"}, + {0x70000000, "SHT_LOPROC"}, + {0x7fffffff, "SHT_HIPROC"}, + {0x80000000, "SHT_LOUSER"}, + {0xffffffff, "SHT_HIUSER"}, +} + +func (i SectionType) String() string { return stringName(uint32(i), shtStrings, false) } +func (i SectionType) GoString() string { return stringName(uint32(i), shtStrings, true) } + +// Section flags. +type SectionFlag uint32 + +const ( + SHF_WRITE SectionFlag = 0x1 /* Section contains writable data. */ + SHF_ALLOC SectionFlag = 0x2 /* Section occupies memory. */ + SHF_EXECINSTR SectionFlag = 0x4 /* Section contains instructions. */ + SHF_MERGE SectionFlag = 0x10 /* Section may be merged. */ + SHF_STRINGS SectionFlag = 0x20 /* Section contains strings. */ + SHF_INFO_LINK SectionFlag = 0x40 /* sh_info holds section index. */ + SHF_LINK_ORDER SectionFlag = 0x80 /* Special ordering requirements. */ + SHF_OS_NONCONFORMING SectionFlag = 0x100 /* OS-specific processing required. */ + SHF_GROUP SectionFlag = 0x200 /* Member of section group. */ + SHF_TLS SectionFlag = 0x400 /* Section contains TLS data. */ + SHF_MASKOS SectionFlag = 0x0ff00000 /* OS-specific semantics. */ + SHF_MASKPROC SectionFlag = 0xf0000000 /* Processor-specific semantics. */ +) + +var shfStrings = []intName{ + {0x1, "SHF_WRITE"}, + {0x2, "SHF_ALLOC"}, + {0x4, "SHF_EXECINSTR"}, + {0x10, "SHF_MERGE"}, + {0x20, "SHF_STRINGS"}, + {0x40, "SHF_INFO_LINK"}, + {0x80, "SHF_LINK_ORDER"}, + {0x100, "SHF_OS_NONCONFORMING"}, + {0x200, "SHF_GROUP"}, + {0x400, "SHF_TLS"}, +} + +func (i SectionFlag) String() string { return flagName(uint32(i), shfStrings, false) } +func (i SectionFlag) GoString() string { return flagName(uint32(i), shfStrings, true) } + +// Prog.Type +type ProgType int + +const ( + PT_NULL ProgType = 0 /* Unused entry. */ + PT_LOAD ProgType = 1 /* Loadable segment. */ + PT_DYNAMIC ProgType = 2 /* Dynamic linking information segment. */ + PT_INTERP ProgType = 3 /* Pathname of interpreter. */ + PT_NOTE ProgType = 4 /* Auxiliary information. */ + PT_SHLIB ProgType = 5 /* Reserved (not used). */ + PT_PHDR ProgType = 6 /* Location of program header itself. */ + PT_TLS ProgType = 7 /* Thread local storage segment */ + PT_LOOS ProgType = 0x60000000 /* First OS-specific. */ + PT_HIOS ProgType = 0x6fffffff /* Last OS-specific. */ + PT_LOPROC ProgType = 0x70000000 /* First processor-specific type. */ + PT_HIPROC ProgType = 0x7fffffff /* Last processor-specific type. */ +) + +var ptStrings = []intName{ + {0, "PT_NULL"}, + {1, "PT_LOAD"}, + {2, "PT_DYNAMIC"}, + {3, "PT_INTERP"}, + {4, "PT_NOTE"}, + {5, "PT_SHLIB"}, + {6, "PT_PHDR"}, + {7, "PT_TLS"}, + {0x60000000, "PT_LOOS"}, + {0x6fffffff, "PT_HIOS"}, + {0x70000000, "PT_LOPROC"}, + {0x7fffffff, "PT_HIPROC"}, +} + +func (i ProgType) String() string { return stringName(uint32(i), ptStrings, false) } +func (i ProgType) GoString() string { return stringName(uint32(i), ptStrings, true) } + +// Prog.Flag +type ProgFlag uint32 + +const ( + PF_X ProgFlag = 0x1 /* Executable. */ + PF_W ProgFlag = 0x2 /* Writable. */ + PF_R ProgFlag = 0x4 /* Readable. */ + PF_MASKOS ProgFlag = 0x0ff00000 /* Operating system-specific. */ + PF_MASKPROC ProgFlag = 0xf0000000 /* Processor-specific. */ +) + +var pfStrings = []intName{ + {0x1, "PF_X"}, + {0x2, "PF_W"}, + {0x4, "PF_R"}, +} + +func (i ProgFlag) String() string { return flagName(uint32(i), pfStrings, false) } +func (i ProgFlag) GoString() string { return flagName(uint32(i), pfStrings, true) } + +// Dyn.Tag +type DynTag int + +const ( + DT_NULL DynTag = 0 /* Terminating entry. */ + DT_NEEDED DynTag = 1 /* String table offset of a needed shared library. */ + DT_PLTRELSZ DynTag = 2 /* Total size in bytes of PLT relocations. */ + DT_PLTGOT DynTag = 3 /* Processor-dependent address. */ + DT_HASH DynTag = 4 /* Address of symbol hash table. */ + DT_STRTAB DynTag = 5 /* Address of string table. */ + DT_SYMTAB DynTag = 6 /* Address of symbol table. */ + DT_RELA DynTag = 7 /* Address of ElfNN_Rela relocations. */ + DT_RELASZ DynTag = 8 /* Total size of ElfNN_Rela relocations. */ + DT_RELAENT DynTag = 9 /* Size of each ElfNN_Rela relocation entry. */ + DT_STRSZ DynTag = 10 /* Size of string table. */ + DT_SYMENT DynTag = 11 /* Size of each symbol table entry. */ + DT_INIT DynTag = 12 /* Address of initialization function. */ + DT_FINI DynTag = 13 /* Address of finalization function. */ + DT_SONAME DynTag = 14 /* String table offset of shared object name. */ + DT_RPATH DynTag = 15 /* String table offset of library path. [sup] */ + DT_SYMBOLIC DynTag = 16 /* Indicates "symbolic" linking. [sup] */ + DT_REL DynTag = 17 /* Address of ElfNN_Rel relocations. */ + DT_RELSZ DynTag = 18 /* Total size of ElfNN_Rel relocations. */ + DT_RELENT DynTag = 19 /* Size of each ElfNN_Rel relocation. */ + DT_PLTREL DynTag = 20 /* Type of relocation used for PLT. */ + DT_DEBUG DynTag = 21 /* Reserved (not used). */ + DT_TEXTREL DynTag = 22 /* Indicates there may be relocations in non-writable segments. [sup] */ + DT_JMPREL DynTag = 23 /* Address of PLT relocations. */ + DT_BIND_NOW DynTag = 24 /* [sup] */ + DT_INIT_ARRAY DynTag = 25 /* Address of the array of pointers to initialization functions */ + DT_FINI_ARRAY DynTag = 26 /* Address of the array of pointers to termination functions */ + DT_INIT_ARRAYSZ DynTag = 27 /* Size in bytes of the array of initialization functions. */ + DT_FINI_ARRAYSZ DynTag = 28 /* Size in bytes of the array of terminationfunctions. */ + DT_RUNPATH DynTag = 29 /* String table offset of a null-terminated library search path string. */ + DT_FLAGS DynTag = 30 /* Object specific flag values. */ + DT_ENCODING DynTag = 32 /* Values greater than or equal to DT_ENCODING + and less than DT_LOOS follow the rules for + the interpretation of the d_un union + as follows: even == 'd_ptr', even == 'd_val' + or none */ + DT_PREINIT_ARRAY DynTag = 32 /* Address of the array of pointers to pre-initialization functions. */ + DT_PREINIT_ARRAYSZ DynTag = 33 /* Size in bytes of the array of pre-initialization functions. */ + DT_LOOS DynTag = 0x6000000d /* First OS-specific */ + DT_HIOS DynTag = 0x6ffff000 /* Last OS-specific */ + DT_LOPROC DynTag = 0x70000000 /* First processor-specific type. */ + DT_HIPROC DynTag = 0x7fffffff /* Last processor-specific type. */ +) + +var dtStrings = []intName{ + {0, "DT_NULL"}, + {1, "DT_NEEDED"}, + {2, "DT_PLTRELSZ"}, + {3, "DT_PLTGOT"}, + {4, "DT_HASH"}, + {5, "DT_STRTAB"}, + {6, "DT_SYMTAB"}, + {7, "DT_RELA"}, + {8, "DT_RELASZ"}, + {9, "DT_RELAENT"}, + {10, "DT_STRSZ"}, + {11, "DT_SYMENT"}, + {12, "DT_INIT"}, + {13, "DT_FINI"}, + {14, "DT_SONAME"}, + {15, "DT_RPATH"}, + {16, "DT_SYMBOLIC"}, + {17, "DT_REL"}, + {18, "DT_RELSZ"}, + {19, "DT_RELENT"}, + {20, "DT_PLTREL"}, + {21, "DT_DEBUG"}, + {22, "DT_TEXTREL"}, + {23, "DT_JMPREL"}, + {24, "DT_BIND_NOW"}, + {25, "DT_INIT_ARRAY"}, + {26, "DT_FINI_ARRAY"}, + {27, "DT_INIT_ARRAYSZ"}, + {28, "DT_FINI_ARRAYSZ"}, + {29, "DT_RUNPATH"}, + {30, "DT_FLAGS"}, + {32, "DT_ENCODING"}, + {32, "DT_PREINIT_ARRAY"}, + {33, "DT_PREINIT_ARRAYSZ"}, + {0x6000000d, "DT_LOOS"}, + {0x6ffff000, "DT_HIOS"}, + {0x70000000, "DT_LOPROC"}, + {0x7fffffff, "DT_HIPROC"}, +} + +func (i DynTag) String() string { return stringName(uint32(i), dtStrings, false) } +func (i DynTag) GoString() string { return stringName(uint32(i), dtStrings, true) } + +// DT_FLAGS values. +type DynFlag int + +const ( + DF_ORIGIN DynFlag = 0x0001 /* Indicates that the object being loaded may + make reference to the + $ORIGIN substitution string */ + DF_SYMBOLIC DynFlag = 0x0002 /* Indicates "symbolic" linking. */ + DF_TEXTREL DynFlag = 0x0004 /* Indicates there may be relocations in non-writable segments. */ + DF_BIND_NOW DynFlag = 0x0008 /* Indicates that the dynamic linker should + process all relocations for the object + containing this entry before transferring + control to the program. */ + DF_STATIC_TLS DynFlag = 0x0010 /* Indicates that the shared object or + executable contains code using a static + thread-local storage scheme. */ +) + +var dflagStrings = []intName{ + {0x0001, "DF_ORIGIN"}, + {0x0002, "DF_SYMBOLIC"}, + {0x0004, "DF_TEXTREL"}, + {0x0008, "DF_BIND_NOW"}, + {0x0010, "DF_STATIC_TLS"}, +} + +func (i DynFlag) String() string { return flagName(uint32(i), dflagStrings, false) } +func (i DynFlag) GoString() string { return flagName(uint32(i), dflagStrings, true) } + +// NType values; used in core files. +type NType int + +const ( + NT_PRSTATUS NType = 1 /* Process status. */ + NT_FPREGSET NType = 2 /* Floating point registers. */ + NT_PRPSINFO NType = 3 /* Process state info. */ +) + +var ntypeStrings = []intName{ + {1, "NT_PRSTATUS"}, + {2, "NT_FPREGSET"}, + {3, "NT_PRPSINFO"}, +} + +func (i NType) String() string { return stringName(uint32(i), ntypeStrings, false) } +func (i NType) GoString() string { return stringName(uint32(i), ntypeStrings, true) } + +/* Symbol Binding - ELFNN_ST_BIND - st_info */ +type SymBind int + +const ( + STB_LOCAL SymBind = 0 /* Local symbol */ + STB_GLOBAL SymBind = 1 /* Global symbol */ + STB_WEAK SymBind = 2 /* like global - lower precedence */ + STB_LOOS SymBind = 10 /* Reserved range for operating system */ + STB_HIOS SymBind = 12 /* specific semantics. */ + STB_LOPROC SymBind = 13 /* reserved range for processor */ + STB_HIPROC SymBind = 15 /* specific semantics. */ +) + +var stbStrings = []intName{ + {0, "STB_LOCAL"}, + {1, "STB_GLOBAL"}, + {2, "STB_WEAK"}, + {10, "STB_LOOS"}, + {12, "STB_HIOS"}, + {13, "STB_LOPROC"}, + {15, "STB_HIPROC"}, +} + +func (i SymBind) String() string { return stringName(uint32(i), stbStrings, false) } +func (i SymBind) GoString() string { return stringName(uint32(i), stbStrings, true) } + +/* Symbol type - ELFNN_ST_TYPE - st_info */ +type SymType int + +const ( + STT_NOTYPE SymType = 0 /* Unspecified type. */ + STT_OBJECT SymType = 1 /* Data object. */ + STT_FUNC SymType = 2 /* Function. */ + STT_SECTION SymType = 3 /* Section. */ + STT_FILE SymType = 4 /* Source file. */ + STT_COMMON SymType = 5 /* Uninitialized common block. */ + STT_TLS SymType = 6 /* TLS object. */ + STT_LOOS SymType = 10 /* Reserved range for operating system */ + STT_HIOS SymType = 12 /* specific semantics. */ + STT_LOPROC SymType = 13 /* reserved range for processor */ + STT_HIPROC SymType = 15 /* specific semantics. */ +) + +var sttStrings = []intName{ + {0, "STT_NOTYPE"}, + {1, "STT_OBJECT"}, + {2, "STT_FUNC"}, + {3, "STT_SECTION"}, + {4, "STT_FILE"}, + {5, "STT_COMMON"}, + {6, "STT_TLS"}, + {10, "STT_LOOS"}, + {12, "STT_HIOS"}, + {13, "STT_LOPROC"}, + {15, "STT_HIPROC"}, +} + +func (i SymType) String() string { return stringName(uint32(i), sttStrings, false) } +func (i SymType) GoString() string { return stringName(uint32(i), sttStrings, true) } + +/* Symbol visibility - ELFNN_ST_VISIBILITY - st_other */ +type SymVis int + +const ( + STV_DEFAULT SymVis = 0x0 /* Default visibility (see binding). */ + STV_INTERNAL SymVis = 0x1 /* Special meaning in relocatable objects. */ + STV_HIDDEN SymVis = 0x2 /* Not visible. */ + STV_PROTECTED SymVis = 0x3 /* Visible but not preemptible. */ +) + +var stvStrings = []intName{ + {0x0, "STV_DEFAULT"}, + {0x1, "STV_INTERNAL"}, + {0x2, "STV_HIDDEN"}, + {0x3, "STV_PROTECTED"}, +} + +func (i SymVis) String() string { return stringName(uint32(i), stvStrings, false) } +func (i SymVis) GoString() string { return stringName(uint32(i), stvStrings, true) } + +/* + * Relocation types. + */ + +// Relocation types for x86-64. +type R_X86_64 int + +const ( + R_X86_64_NONE R_X86_64 = 0 /* No relocation. */ + R_X86_64_64 R_X86_64 = 1 /* Add 64 bit symbol value. */ + R_X86_64_PC32 R_X86_64 = 2 /* PC-relative 32 bit signed sym value. */ + R_X86_64_GOT32 R_X86_64 = 3 /* PC-relative 32 bit GOT offset. */ + R_X86_64_PLT32 R_X86_64 = 4 /* PC-relative 32 bit PLT offset. */ + R_X86_64_COPY R_X86_64 = 5 /* Copy data from shared object. */ + R_X86_64_GLOB_DAT R_X86_64 = 6 /* Set GOT entry to data address. */ + R_X86_64_JMP_SLOT R_X86_64 = 7 /* Set GOT entry to code address. */ + R_X86_64_RELATIVE R_X86_64 = 8 /* Add load address of shared object. */ + R_X86_64_GOTPCREL R_X86_64 = 9 /* Add 32 bit signed pcrel offset to GOT. */ + R_X86_64_32 R_X86_64 = 10 /* Add 32 bit zero extended symbol value */ + R_X86_64_32S R_X86_64 = 11 /* Add 32 bit sign extended symbol value */ + R_X86_64_16 R_X86_64 = 12 /* Add 16 bit zero extended symbol value */ + R_X86_64_PC16 R_X86_64 = 13 /* Add 16 bit signed extended pc relative symbol value */ + R_X86_64_8 R_X86_64 = 14 /* Add 8 bit zero extended symbol value */ + R_X86_64_PC8 R_X86_64 = 15 /* Add 8 bit signed extended pc relative symbol value */ + R_X86_64_DTPMOD64 R_X86_64 = 16 /* ID of module containing symbol */ + R_X86_64_DTPOFF64 R_X86_64 = 17 /* Offset in TLS block */ + R_X86_64_TPOFF64 R_X86_64 = 18 /* Offset in static TLS block */ + R_X86_64_TLSGD R_X86_64 = 19 /* PC relative offset to GD GOT entry */ + R_X86_64_TLSLD R_X86_64 = 20 /* PC relative offset to LD GOT entry */ + R_X86_64_DTPOFF32 R_X86_64 = 21 /* Offset in TLS block */ + R_X86_64_GOTTPOFF R_X86_64 = 22 /* PC relative offset to IE GOT entry */ + R_X86_64_TPOFF32 R_X86_64 = 23 /* Offset in static TLS block */ +) + +var rx86_64Strings = []intName{ + {0, "R_X86_64_NONE"}, + {1, "R_X86_64_64"}, + {2, "R_X86_64_PC32"}, + {3, "R_X86_64_GOT32"}, + {4, "R_X86_64_PLT32"}, + {5, "R_X86_64_COPY"}, + {6, "R_X86_64_GLOB_DAT"}, + {7, "R_X86_64_JMP_SLOT"}, + {8, "R_X86_64_RELATIVE"}, + {9, "R_X86_64_GOTPCREL"}, + {10, "R_X86_64_32"}, + {11, "R_X86_64_32S"}, + {12, "R_X86_64_16"}, + {13, "R_X86_64_PC16"}, + {14, "R_X86_64_8"}, + {15, "R_X86_64_PC8"}, + {16, "R_X86_64_DTPMOD64"}, + {17, "R_X86_64_DTPOFF64"}, + {18, "R_X86_64_TPOFF64"}, + {19, "R_X86_64_TLSGD"}, + {20, "R_X86_64_TLSLD"}, + {21, "R_X86_64_DTPOFF32"}, + {22, "R_X86_64_GOTTPOFF"}, + {23, "R_X86_64_TPOFF32"}, +} + +func (i R_X86_64) String() string { return stringName(uint32(i), rx86_64Strings, false) } +func (i R_X86_64) GoString() string { return stringName(uint32(i), rx86_64Strings, true) } + +// Relocation types for Alpha. +type R_ALPHA int + +const ( + R_ALPHA_NONE R_ALPHA = 0 /* No reloc */ + R_ALPHA_REFLONG R_ALPHA = 1 /* Direct 32 bit */ + R_ALPHA_REFQUAD R_ALPHA = 2 /* Direct 64 bit */ + R_ALPHA_GPREL32 R_ALPHA = 3 /* GP relative 32 bit */ + R_ALPHA_LITERAL R_ALPHA = 4 /* GP relative 16 bit w/optimization */ + R_ALPHA_LITUSE R_ALPHA = 5 /* Optimization hint for LITERAL */ + R_ALPHA_GPDISP R_ALPHA = 6 /* Add displacement to GP */ + R_ALPHA_BRADDR R_ALPHA = 7 /* PC+4 relative 23 bit shifted */ + R_ALPHA_HINT R_ALPHA = 8 /* PC+4 relative 16 bit shifted */ + R_ALPHA_SREL16 R_ALPHA = 9 /* PC relative 16 bit */ + R_ALPHA_SREL32 R_ALPHA = 10 /* PC relative 32 bit */ + R_ALPHA_SREL64 R_ALPHA = 11 /* PC relative 64 bit */ + R_ALPHA_OP_PUSH R_ALPHA = 12 /* OP stack push */ + R_ALPHA_OP_STORE R_ALPHA = 13 /* OP stack pop and store */ + R_ALPHA_OP_PSUB R_ALPHA = 14 /* OP stack subtract */ + R_ALPHA_OP_PRSHIFT R_ALPHA = 15 /* OP stack right shift */ + R_ALPHA_GPVALUE R_ALPHA = 16 + R_ALPHA_GPRELHIGH R_ALPHA = 17 + R_ALPHA_GPRELLOW R_ALPHA = 18 + R_ALPHA_IMMED_GP_16 R_ALPHA = 19 + R_ALPHA_IMMED_GP_HI32 R_ALPHA = 20 + R_ALPHA_IMMED_SCN_HI32 R_ALPHA = 21 + R_ALPHA_IMMED_BR_HI32 R_ALPHA = 22 + R_ALPHA_IMMED_LO32 R_ALPHA = 23 + R_ALPHA_COPY R_ALPHA = 24 /* Copy symbol at runtime */ + R_ALPHA_GLOB_DAT R_ALPHA = 25 /* Create GOT entry */ + R_ALPHA_JMP_SLOT R_ALPHA = 26 /* Create PLT entry */ + R_ALPHA_RELATIVE R_ALPHA = 27 /* Adjust by program base */ +) + +var ralphaStrings = []intName{ + {0, "R_ALPHA_NONE"}, + {1, "R_ALPHA_REFLONG"}, + {2, "R_ALPHA_REFQUAD"}, + {3, "R_ALPHA_GPREL32"}, + {4, "R_ALPHA_LITERAL"}, + {5, "R_ALPHA_LITUSE"}, + {6, "R_ALPHA_GPDISP"}, + {7, "R_ALPHA_BRADDR"}, + {8, "R_ALPHA_HINT"}, + {9, "R_ALPHA_SREL16"}, + {10, "R_ALPHA_SREL32"}, + {11, "R_ALPHA_SREL64"}, + {12, "R_ALPHA_OP_PUSH"}, + {13, "R_ALPHA_OP_STORE"}, + {14, "R_ALPHA_OP_PSUB"}, + {15, "R_ALPHA_OP_PRSHIFT"}, + {16, "R_ALPHA_GPVALUE"}, + {17, "R_ALPHA_GPRELHIGH"}, + {18, "R_ALPHA_GPRELLOW"}, + {19, "R_ALPHA_IMMED_GP_16"}, + {20, "R_ALPHA_IMMED_GP_HI32"}, + {21, "R_ALPHA_IMMED_SCN_HI32"}, + {22, "R_ALPHA_IMMED_BR_HI32"}, + {23, "R_ALPHA_IMMED_LO32"}, + {24, "R_ALPHA_COPY"}, + {25, "R_ALPHA_GLOB_DAT"}, + {26, "R_ALPHA_JMP_SLOT"}, + {27, "R_ALPHA_RELATIVE"}, +} + +func (i R_ALPHA) String() string { return stringName(uint32(i), ralphaStrings, false) } +func (i R_ALPHA) GoString() string { return stringName(uint32(i), ralphaStrings, true) } + +// Relocation types for ARM. +type R_ARM int + +const ( + R_ARM_NONE R_ARM = 0 /* No relocation. */ + R_ARM_PC24 R_ARM = 1 + R_ARM_ABS32 R_ARM = 2 + R_ARM_REL32 R_ARM = 3 + R_ARM_PC13 R_ARM = 4 + R_ARM_ABS16 R_ARM = 5 + R_ARM_ABS12 R_ARM = 6 + R_ARM_THM_ABS5 R_ARM = 7 + R_ARM_ABS8 R_ARM = 8 + R_ARM_SBREL32 R_ARM = 9 + R_ARM_THM_PC22 R_ARM = 10 + R_ARM_THM_PC8 R_ARM = 11 + R_ARM_AMP_VCALL9 R_ARM = 12 + R_ARM_SWI24 R_ARM = 13 + R_ARM_THM_SWI8 R_ARM = 14 + R_ARM_XPC25 R_ARM = 15 + R_ARM_THM_XPC22 R_ARM = 16 + R_ARM_COPY R_ARM = 20 /* Copy data from shared object. */ + R_ARM_GLOB_DAT R_ARM = 21 /* Set GOT entry to data address. */ + R_ARM_JUMP_SLOT R_ARM = 22 /* Set GOT entry to code address. */ + R_ARM_RELATIVE R_ARM = 23 /* Add load address of shared object. */ + R_ARM_GOTOFF R_ARM = 24 /* Add GOT-relative symbol address. */ + R_ARM_GOTPC R_ARM = 25 /* Add PC-relative GOT table address. */ + R_ARM_GOT32 R_ARM = 26 /* Add PC-relative GOT offset. */ + R_ARM_PLT32 R_ARM = 27 /* Add PC-relative PLT offset. */ + R_ARM_GNU_VTENTRY R_ARM = 100 + R_ARM_GNU_VTINHERIT R_ARM = 101 + R_ARM_RSBREL32 R_ARM = 250 + R_ARM_THM_RPC22 R_ARM = 251 + R_ARM_RREL32 R_ARM = 252 + R_ARM_RABS32 R_ARM = 253 + R_ARM_RPC24 R_ARM = 254 + R_ARM_RBASE R_ARM = 255 +) + +var rarmStrings = []intName{ + {0, "R_ARM_NONE"}, + {1, "R_ARM_PC24"}, + {2, "R_ARM_ABS32"}, + {3, "R_ARM_REL32"}, + {4, "R_ARM_PC13"}, + {5, "R_ARM_ABS16"}, + {6, "R_ARM_ABS12"}, + {7, "R_ARM_THM_ABS5"}, + {8, "R_ARM_ABS8"}, + {9, "R_ARM_SBREL32"}, + {10, "R_ARM_THM_PC22"}, + {11, "R_ARM_THM_PC8"}, + {12, "R_ARM_AMP_VCALL9"}, + {13, "R_ARM_SWI24"}, + {14, "R_ARM_THM_SWI8"}, + {15, "R_ARM_XPC25"}, + {16, "R_ARM_THM_XPC22"}, + {20, "R_ARM_COPY"}, + {21, "R_ARM_GLOB_DAT"}, + {22, "R_ARM_JUMP_SLOT"}, + {23, "R_ARM_RELATIVE"}, + {24, "R_ARM_GOTOFF"}, + {25, "R_ARM_GOTPC"}, + {26, "R_ARM_GOT32"}, + {27, "R_ARM_PLT32"}, + {100, "R_ARM_GNU_VTENTRY"}, + {101, "R_ARM_GNU_VTINHERIT"}, + {250, "R_ARM_RSBREL32"}, + {251, "R_ARM_THM_RPC22"}, + {252, "R_ARM_RREL32"}, + {253, "R_ARM_RABS32"}, + {254, "R_ARM_RPC24"}, + {255, "R_ARM_RBASE"}, +} + +func (i R_ARM) String() string { return stringName(uint32(i), rarmStrings, false) } +func (i R_ARM) GoString() string { return stringName(uint32(i), rarmStrings, true) } + +// Relocation types for 386. +type R_386 int + +const ( + R_386_NONE R_386 = 0 /* No relocation. */ + R_386_32 R_386 = 1 /* Add symbol value. */ + R_386_PC32 R_386 = 2 /* Add PC-relative symbol value. */ + R_386_GOT32 R_386 = 3 /* Add PC-relative GOT offset. */ + R_386_PLT32 R_386 = 4 /* Add PC-relative PLT offset. */ + R_386_COPY R_386 = 5 /* Copy data from shared object. */ + R_386_GLOB_DAT R_386 = 6 /* Set GOT entry to data address. */ + R_386_JMP_SLOT R_386 = 7 /* Set GOT entry to code address. */ + R_386_RELATIVE R_386 = 8 /* Add load address of shared object. */ + R_386_GOTOFF R_386 = 9 /* Add GOT-relative symbol address. */ + R_386_GOTPC R_386 = 10 /* Add PC-relative GOT table address. */ + R_386_TLS_TPOFF R_386 = 14 /* Negative offset in static TLS block */ + R_386_TLS_IE R_386 = 15 /* Absolute address of GOT for -ve static TLS */ + R_386_TLS_GOTIE R_386 = 16 /* GOT entry for negative static TLS block */ + R_386_TLS_LE R_386 = 17 /* Negative offset relative to static TLS */ + R_386_TLS_GD R_386 = 18 /* 32 bit offset to GOT (index,off) pair */ + R_386_TLS_LDM R_386 = 19 /* 32 bit offset to GOT (index,zero) pair */ + R_386_TLS_GD_32 R_386 = 24 /* 32 bit offset to GOT (index,off) pair */ + R_386_TLS_GD_PUSH R_386 = 25 /* pushl instruction for Sun ABI GD sequence */ + R_386_TLS_GD_CALL R_386 = 26 /* call instruction for Sun ABI GD sequence */ + R_386_TLS_GD_POP R_386 = 27 /* popl instruction for Sun ABI GD sequence */ + R_386_TLS_LDM_32 R_386 = 28 /* 32 bit offset to GOT (index,zero) pair */ + R_386_TLS_LDM_PUSH R_386 = 29 /* pushl instruction for Sun ABI LD sequence */ + R_386_TLS_LDM_CALL R_386 = 30 /* call instruction for Sun ABI LD sequence */ + R_386_TLS_LDM_POP R_386 = 31 /* popl instruction for Sun ABI LD sequence */ + R_386_TLS_LDO_32 R_386 = 32 /* 32 bit offset from start of TLS block */ + R_386_TLS_IE_32 R_386 = 33 /* 32 bit offset to GOT static TLS offset entry */ + R_386_TLS_LE_32 R_386 = 34 /* 32 bit offset within static TLS block */ + R_386_TLS_DTPMOD32 R_386 = 35 /* GOT entry containing TLS index */ + R_386_TLS_DTPOFF32 R_386 = 36 /* GOT entry containing TLS offset */ + R_386_TLS_TPOFF32 R_386 = 37 /* GOT entry of -ve static TLS offset */ +) + +var r386Strings = []intName{ + {0, "R_386_NONE"}, + {1, "R_386_32"}, + {2, "R_386_PC32"}, + {3, "R_386_GOT32"}, + {4, "R_386_PLT32"}, + {5, "R_386_COPY"}, + {6, "R_386_GLOB_DAT"}, + {7, "R_386_JMP_SLOT"}, + {8, "R_386_RELATIVE"}, + {9, "R_386_GOTOFF"}, + {10, "R_386_GOTPC"}, + {14, "R_386_TLS_TPOFF"}, + {15, "R_386_TLS_IE"}, + {16, "R_386_TLS_GOTIE"}, + {17, "R_386_TLS_LE"}, + {18, "R_386_TLS_GD"}, + {19, "R_386_TLS_LDM"}, + {24, "R_386_TLS_GD_32"}, + {25, "R_386_TLS_GD_PUSH"}, + {26, "R_386_TLS_GD_CALL"}, + {27, "R_386_TLS_GD_POP"}, + {28, "R_386_TLS_LDM_32"}, + {29, "R_386_TLS_LDM_PUSH"}, + {30, "R_386_TLS_LDM_CALL"}, + {31, "R_386_TLS_LDM_POP"}, + {32, "R_386_TLS_LDO_32"}, + {33, "R_386_TLS_IE_32"}, + {34, "R_386_TLS_LE_32"}, + {35, "R_386_TLS_DTPMOD32"}, + {36, "R_386_TLS_DTPOFF32"}, + {37, "R_386_TLS_TPOFF32"}, +} + +func (i R_386) String() string { return stringName(uint32(i), r386Strings, false) } +func (i R_386) GoString() string { return stringName(uint32(i), r386Strings, true) } + +// Relocation types for PowerPC. +type R_PPC int + +const ( + R_PPC_NONE R_PPC = 0 /* No relocation. */ + R_PPC_ADDR32 R_PPC = 1 + R_PPC_ADDR24 R_PPC = 2 + R_PPC_ADDR16 R_PPC = 3 + R_PPC_ADDR16_LO R_PPC = 4 + R_PPC_ADDR16_HI R_PPC = 5 + R_PPC_ADDR16_HA R_PPC = 6 + R_PPC_ADDR14 R_PPC = 7 + R_PPC_ADDR14_BRTAKEN R_PPC = 8 + R_PPC_ADDR14_BRNTAKEN R_PPC = 9 + R_PPC_REL24 R_PPC = 10 + R_PPC_REL14 R_PPC = 11 + R_PPC_REL14_BRTAKEN R_PPC = 12 + R_PPC_REL14_BRNTAKEN R_PPC = 13 + R_PPC_GOT16 R_PPC = 14 + R_PPC_GOT16_LO R_PPC = 15 + R_PPC_GOT16_HI R_PPC = 16 + R_PPC_GOT16_HA R_PPC = 17 + R_PPC_PLTREL24 R_PPC = 18 + R_PPC_COPY R_PPC = 19 + R_PPC_GLOB_DAT R_PPC = 20 + R_PPC_JMP_SLOT R_PPC = 21 + R_PPC_RELATIVE R_PPC = 22 + R_PPC_LOCAL24PC R_PPC = 23 + R_PPC_UADDR32 R_PPC = 24 + R_PPC_UADDR16 R_PPC = 25 + R_PPC_REL32 R_PPC = 26 + R_PPC_PLT32 R_PPC = 27 + R_PPC_PLTREL32 R_PPC = 28 + R_PPC_PLT16_LO R_PPC = 29 + R_PPC_PLT16_HI R_PPC = 30 + R_PPC_PLT16_HA R_PPC = 31 + R_PPC_SDAREL16 R_PPC = 32 + R_PPC_SECTOFF R_PPC = 33 + R_PPC_SECTOFF_LO R_PPC = 34 + R_PPC_SECTOFF_HI R_PPC = 35 + R_PPC_SECTOFF_HA R_PPC = 36 + R_PPC_TLS R_PPC = 67 + R_PPC_DTPMOD32 R_PPC = 68 + R_PPC_TPREL16 R_PPC = 69 + R_PPC_TPREL16_LO R_PPC = 70 + R_PPC_TPREL16_HI R_PPC = 71 + R_PPC_TPREL16_HA R_PPC = 72 + R_PPC_TPREL32 R_PPC = 73 + R_PPC_DTPREL16 R_PPC = 74 + R_PPC_DTPREL16_LO R_PPC = 75 + R_PPC_DTPREL16_HI R_PPC = 76 + R_PPC_DTPREL16_HA R_PPC = 77 + R_PPC_DTPREL32 R_PPC = 78 + R_PPC_GOT_TLSGD16 R_PPC = 79 + R_PPC_GOT_TLSGD16_LO R_PPC = 80 + R_PPC_GOT_TLSGD16_HI R_PPC = 81 + R_PPC_GOT_TLSGD16_HA R_PPC = 82 + R_PPC_GOT_TLSLD16 R_PPC = 83 + R_PPC_GOT_TLSLD16_LO R_PPC = 84 + R_PPC_GOT_TLSLD16_HI R_PPC = 85 + R_PPC_GOT_TLSLD16_HA R_PPC = 86 + R_PPC_GOT_TPREL16 R_PPC = 87 + R_PPC_GOT_TPREL16_LO R_PPC = 88 + R_PPC_GOT_TPREL16_HI R_PPC = 89 + R_PPC_GOT_TPREL16_HA R_PPC = 90 + R_PPC_EMB_NADDR32 R_PPC = 101 + R_PPC_EMB_NADDR16 R_PPC = 102 + R_PPC_EMB_NADDR16_LO R_PPC = 103 + R_PPC_EMB_NADDR16_HI R_PPC = 104 + R_PPC_EMB_NADDR16_HA R_PPC = 105 + R_PPC_EMB_SDAI16 R_PPC = 106 + R_PPC_EMB_SDA2I16 R_PPC = 107 + R_PPC_EMB_SDA2REL R_PPC = 108 + R_PPC_EMB_SDA21 R_PPC = 109 + R_PPC_EMB_MRKREF R_PPC = 110 + R_PPC_EMB_RELSEC16 R_PPC = 111 + R_PPC_EMB_RELST_LO R_PPC = 112 + R_PPC_EMB_RELST_HI R_PPC = 113 + R_PPC_EMB_RELST_HA R_PPC = 114 + R_PPC_EMB_BIT_FLD R_PPC = 115 + R_PPC_EMB_RELSDA R_PPC = 116 +) + +var rppcStrings = []intName{ + {0, "R_PPC_NONE"}, + {1, "R_PPC_ADDR32"}, + {2, "R_PPC_ADDR24"}, + {3, "R_PPC_ADDR16"}, + {4, "R_PPC_ADDR16_LO"}, + {5, "R_PPC_ADDR16_HI"}, + {6, "R_PPC_ADDR16_HA"}, + {7, "R_PPC_ADDR14"}, + {8, "R_PPC_ADDR14_BRTAKEN"}, + {9, "R_PPC_ADDR14_BRNTAKEN"}, + {10, "R_PPC_REL24"}, + {11, "R_PPC_REL14"}, + {12, "R_PPC_REL14_BRTAKEN"}, + {13, "R_PPC_REL14_BRNTAKEN"}, + {14, "R_PPC_GOT16"}, + {15, "R_PPC_GOT16_LO"}, + {16, "R_PPC_GOT16_HI"}, + {17, "R_PPC_GOT16_HA"}, + {18, "R_PPC_PLTREL24"}, + {19, "R_PPC_COPY"}, + {20, "R_PPC_GLOB_DAT"}, + {21, "R_PPC_JMP_SLOT"}, + {22, "R_PPC_RELATIVE"}, + {23, "R_PPC_LOCAL24PC"}, + {24, "R_PPC_UADDR32"}, + {25, "R_PPC_UADDR16"}, + {26, "R_PPC_REL32"}, + {27, "R_PPC_PLT32"}, + {28, "R_PPC_PLTREL32"}, + {29, "R_PPC_PLT16_LO"}, + {30, "R_PPC_PLT16_HI"}, + {31, "R_PPC_PLT16_HA"}, + {32, "R_PPC_SDAREL16"}, + {33, "R_PPC_SECTOFF"}, + {34, "R_PPC_SECTOFF_LO"}, + {35, "R_PPC_SECTOFF_HI"}, + {36, "R_PPC_SECTOFF_HA"}, + + {67, "R_PPC_TLS"}, + {68, "R_PPC_DTPMOD32"}, + {69, "R_PPC_TPREL16"}, + {70, "R_PPC_TPREL16_LO"}, + {71, "R_PPC_TPREL16_HI"}, + {72, "R_PPC_TPREL16_HA"}, + {73, "R_PPC_TPREL32"}, + {74, "R_PPC_DTPREL16"}, + {75, "R_PPC_DTPREL16_LO"}, + {76, "R_PPC_DTPREL16_HI"}, + {77, "R_PPC_DTPREL16_HA"}, + {78, "R_PPC_DTPREL32"}, + {79, "R_PPC_GOT_TLSGD16"}, + {80, "R_PPC_GOT_TLSGD16_LO"}, + {81, "R_PPC_GOT_TLSGD16_HI"}, + {82, "R_PPC_GOT_TLSGD16_HA"}, + {83, "R_PPC_GOT_TLSLD16"}, + {84, "R_PPC_GOT_TLSLD16_LO"}, + {85, "R_PPC_GOT_TLSLD16_HI"}, + {86, "R_PPC_GOT_TLSLD16_HA"}, + {87, "R_PPC_GOT_TPREL16"}, + {88, "R_PPC_GOT_TPREL16_LO"}, + {89, "R_PPC_GOT_TPREL16_HI"}, + {90, "R_PPC_GOT_TPREL16_HA"}, + + {101, "R_PPC_EMB_NADDR32"}, + {102, "R_PPC_EMB_NADDR16"}, + {103, "R_PPC_EMB_NADDR16_LO"}, + {104, "R_PPC_EMB_NADDR16_HI"}, + {105, "R_PPC_EMB_NADDR16_HA"}, + {106, "R_PPC_EMB_SDAI16"}, + {107, "R_PPC_EMB_SDA2I16"}, + {108, "R_PPC_EMB_SDA2REL"}, + {109, "R_PPC_EMB_SDA21"}, + {110, "R_PPC_EMB_MRKREF"}, + {111, "R_PPC_EMB_RELSEC16"}, + {112, "R_PPC_EMB_RELST_LO"}, + {113, "R_PPC_EMB_RELST_HI"}, + {114, "R_PPC_EMB_RELST_HA"}, + {115, "R_PPC_EMB_BIT_FLD"}, + {116, "R_PPC_EMB_RELSDA"}, +} + +func (i R_PPC) String() string { return stringName(uint32(i), rppcStrings, false) } +func (i R_PPC) GoString() string { return stringName(uint32(i), rppcStrings, true) } + +// Relocation types for SPARC. +type R_SPARC int + +const ( + R_SPARC_NONE R_SPARC = 0 + R_SPARC_8 R_SPARC = 1 + R_SPARC_16 R_SPARC = 2 + R_SPARC_32 R_SPARC = 3 + R_SPARC_DISP8 R_SPARC = 4 + R_SPARC_DISP16 R_SPARC = 5 + R_SPARC_DISP32 R_SPARC = 6 + R_SPARC_WDISP30 R_SPARC = 7 + R_SPARC_WDISP22 R_SPARC = 8 + R_SPARC_HI22 R_SPARC = 9 + R_SPARC_22 R_SPARC = 10 + R_SPARC_13 R_SPARC = 11 + R_SPARC_LO10 R_SPARC = 12 + R_SPARC_GOT10 R_SPARC = 13 + R_SPARC_GOT13 R_SPARC = 14 + R_SPARC_GOT22 R_SPARC = 15 + R_SPARC_PC10 R_SPARC = 16 + R_SPARC_PC22 R_SPARC = 17 + R_SPARC_WPLT30 R_SPARC = 18 + R_SPARC_COPY R_SPARC = 19 + R_SPARC_GLOB_DAT R_SPARC = 20 + R_SPARC_JMP_SLOT R_SPARC = 21 + R_SPARC_RELATIVE R_SPARC = 22 + R_SPARC_UA32 R_SPARC = 23 + R_SPARC_PLT32 R_SPARC = 24 + R_SPARC_HIPLT22 R_SPARC = 25 + R_SPARC_LOPLT10 R_SPARC = 26 + R_SPARC_PCPLT32 R_SPARC = 27 + R_SPARC_PCPLT22 R_SPARC = 28 + R_SPARC_PCPLT10 R_SPARC = 29 + R_SPARC_10 R_SPARC = 30 + R_SPARC_11 R_SPARC = 31 + R_SPARC_64 R_SPARC = 32 + R_SPARC_OLO10 R_SPARC = 33 + R_SPARC_HH22 R_SPARC = 34 + R_SPARC_HM10 R_SPARC = 35 + R_SPARC_LM22 R_SPARC = 36 + R_SPARC_PC_HH22 R_SPARC = 37 + R_SPARC_PC_HM10 R_SPARC = 38 + R_SPARC_PC_LM22 R_SPARC = 39 + R_SPARC_WDISP16 R_SPARC = 40 + R_SPARC_WDISP19 R_SPARC = 41 + R_SPARC_GLOB_JMP R_SPARC = 42 + R_SPARC_7 R_SPARC = 43 + R_SPARC_5 R_SPARC = 44 + R_SPARC_6 R_SPARC = 45 + R_SPARC_DISP64 R_SPARC = 46 + R_SPARC_PLT64 R_SPARC = 47 + R_SPARC_HIX22 R_SPARC = 48 + R_SPARC_LOX10 R_SPARC = 49 + R_SPARC_H44 R_SPARC = 50 + R_SPARC_M44 R_SPARC = 51 + R_SPARC_L44 R_SPARC = 52 + R_SPARC_REGISTER R_SPARC = 53 + R_SPARC_UA64 R_SPARC = 54 + R_SPARC_UA16 R_SPARC = 55 +) + +var rsparcStrings = []intName{ + {0, "R_SPARC_NONE"}, + {1, "R_SPARC_8"}, + {2, "R_SPARC_16"}, + {3, "R_SPARC_32"}, + {4, "R_SPARC_DISP8"}, + {5, "R_SPARC_DISP16"}, + {6, "R_SPARC_DISP32"}, + {7, "R_SPARC_WDISP30"}, + {8, "R_SPARC_WDISP22"}, + {9, "R_SPARC_HI22"}, + {10, "R_SPARC_22"}, + {11, "R_SPARC_13"}, + {12, "R_SPARC_LO10"}, + {13, "R_SPARC_GOT10"}, + {14, "R_SPARC_GOT13"}, + {15, "R_SPARC_GOT22"}, + {16, "R_SPARC_PC10"}, + {17, "R_SPARC_PC22"}, + {18, "R_SPARC_WPLT30"}, + {19, "R_SPARC_COPY"}, + {20, "R_SPARC_GLOB_DAT"}, + {21, "R_SPARC_JMP_SLOT"}, + {22, "R_SPARC_RELATIVE"}, + {23, "R_SPARC_UA32"}, + {24, "R_SPARC_PLT32"}, + {25, "R_SPARC_HIPLT22"}, + {26, "R_SPARC_LOPLT10"}, + {27, "R_SPARC_PCPLT32"}, + {28, "R_SPARC_PCPLT22"}, + {29, "R_SPARC_PCPLT10"}, + {30, "R_SPARC_10"}, + {31, "R_SPARC_11"}, + {32, "R_SPARC_64"}, + {33, "R_SPARC_OLO10"}, + {34, "R_SPARC_HH22"}, + {35, "R_SPARC_HM10"}, + {36, "R_SPARC_LM22"}, + {37, "R_SPARC_PC_HH22"}, + {38, "R_SPARC_PC_HM10"}, + {39, "R_SPARC_PC_LM22"}, + {40, "R_SPARC_WDISP16"}, + {41, "R_SPARC_WDISP19"}, + {42, "R_SPARC_GLOB_JMP"}, + {43, "R_SPARC_7"}, + {44, "R_SPARC_5"}, + {45, "R_SPARC_6"}, + {46, "R_SPARC_DISP64"}, + {47, "R_SPARC_PLT64"}, + {48, "R_SPARC_HIX22"}, + {49, "R_SPARC_LOX10"}, + {50, "R_SPARC_H44"}, + {51, "R_SPARC_M44"}, + {52, "R_SPARC_L44"}, + {53, "R_SPARC_REGISTER"}, + {54, "R_SPARC_UA64"}, + {55, "R_SPARC_UA16"}, +} + +func (i R_SPARC) String() string { return stringName(uint32(i), rsparcStrings, false) } +func (i R_SPARC) GoString() string { return stringName(uint32(i), rsparcStrings, true) } + +// Magic number for the elf trampoline, chosen wisely to be an immediate value. +const ARM_MAGIC_TRAMP_NUMBER = 0x5c000003 + + +// ELF32 File header. +type Header32 struct { + Ident [EI_NIDENT]byte /* File identification. */ + Type uint16 /* File type. */ + Machine uint16 /* Machine architecture. */ + Version uint32 /* ELF format version. */ + Entry uint32 /* Entry point. */ + Phoff uint32 /* Program header file offset. */ + Shoff uint32 /* Section header file offset. */ + Flags uint32 /* Architecture-specific flags. */ + Ehsize uint16 /* Size of ELF header in bytes. */ + Phentsize uint16 /* Size of program header entry. */ + Phnum uint16 /* Number of program header entries. */ + Shentsize uint16 /* Size of section header entry. */ + Shnum uint16 /* Number of section header entries. */ + Shstrndx uint16 /* Section name strings section. */ +} + +// ELF32 Section header. +type Section32 struct { + Name uint32 /* Section name (index into the section header string table). */ + Type uint32 /* Section type. */ + Flags uint32 /* Section flags. */ + Addr uint32 /* Address in memory image. */ + Off uint32 /* Offset in file. */ + Size uint32 /* Size in bytes. */ + Link uint32 /* Index of a related section. */ + Info uint32 /* Depends on section type. */ + Addralign uint32 /* Alignment in bytes. */ + Entsize uint32 /* Size of each entry in section. */ +} + +// ELF32 Program header. +type Prog32 struct { + Type uint32 /* Entry type. */ + Off uint32 /* File offset of contents. */ + Vaddr uint32 /* Virtual address in memory image. */ + Paddr uint32 /* Physical address (not used). */ + Filesz uint32 /* Size of contents in file. */ + Memsz uint32 /* Size of contents in memory. */ + Flags uint32 /* Access permission flags. */ + Align uint32 /* Alignment in memory and file. */ +} + +// ELF32 Dynamic structure. The ".dynamic" section contains an array of them. +type Dyn32 struct { + Tag int32 /* Entry type. */ + Val uint32 /* Integer/Address value. */ +} + +/* + * Relocation entries. + */ + +// ELF32 Relocations that don't need an addend field. +type Rel32 struct { + Off uint32 /* Location to be relocated. */ + Info uint32 /* Relocation type and symbol index. */ +} + +// ELF32 Relocations that need an addend field. +type Rela32 struct { + Off uint32 /* Location to be relocated. */ + Info uint32 /* Relocation type and symbol index. */ + Addend int32 /* Addend. */ +} + +func R_SYM32(info uint32) uint32 { return uint32(info >> 8) } +func R_TYPE32(info uint32) uint32 { return uint32(info & 0xff) } +func R_INFO32(sym, typ uint32) uint32 { return sym<<8 | typ } + +// ELF32 Symbol. +type Sym32 struct { + Name uint32 + Value uint32 + Size uint32 + Info uint8 + Other uint8 + Shndx uint16 +} + +const Sym32Size = 16 + +func ST_BIND(info uint8) SymBind { return SymBind(info >> 4) } +func ST_TYPE(info uint8) SymType { return SymType(info & 0xF) } +func ST_INFO(bind SymBind, typ SymType) uint8 { + return uint8(bind)<<4 | uint8(typ)&0xf +} +func ST_VISIBILITY(other uint8) SymVis { return SymVis(other & 3) } + +/* + * ELF64 + */ + +// ELF64 file header. +type Header64 struct { + Ident [EI_NIDENT]byte /* File identification. */ + Type uint16 /* File type. */ + Machine uint16 /* Machine architecture. */ + Version uint32 /* ELF format version. */ + Entry uint64 /* Entry point. */ + Phoff uint64 /* Program header file offset. */ + Shoff uint64 /* Section header file offset. */ + Flags uint32 /* Architecture-specific flags. */ + Ehsize uint16 /* Size of ELF header in bytes. */ + Phentsize uint16 /* Size of program header entry. */ + Phnum uint16 /* Number of program header entries. */ + Shentsize uint16 /* Size of section header entry. */ + Shnum uint16 /* Number of section header entries. */ + Shstrndx uint16 /* Section name strings section. */ +} + +// ELF64 Section header. +type Section64 struct { + Name uint32 /* Section name (index into the section header string table). */ + Type uint32 /* Section type. */ + Flags uint64 /* Section flags. */ + Addr uint64 /* Address in memory image. */ + Off uint64 /* Offset in file. */ + Size uint64 /* Size in bytes. */ + Link uint32 /* Index of a related section. */ + Info uint32 /* Depends on section type. */ + Addralign uint64 /* Alignment in bytes. */ + Entsize uint64 /* Size of each entry in section. */ +} + +// ELF64 Program header. +type Prog64 struct { + Type uint32 /* Entry type. */ + Flags uint32 /* Access permission flags. */ + Off uint64 /* File offset of contents. */ + Vaddr uint64 /* Virtual address in memory image. */ + Paddr uint64 /* Physical address (not used). */ + Filesz uint64 /* Size of contents in file. */ + Memsz uint64 /* Size of contents in memory. */ + Align uint64 /* Alignment in memory and file. */ +} + +// ELF64 Dynamic structure. The ".dynamic" section contains an array of them. +type Dyn64 struct { + Tag int64 /* Entry type. */ + Val uint64 /* Integer/address value */ +} + +/* + * Relocation entries. + */ + +/* ELF64 relocations that don't need an addend field. */ +type Rel64 struct { + Off uint64 /* Location to be relocated. */ + Info uint64 /* Relocation type and symbol index. */ +} + +/* ELF64 relocations that need an addend field. */ +type Rela64 struct { + Off uint64 /* Location to be relocated. */ + Info uint64 /* Relocation type and symbol index. */ + Addend int64 /* Addend. */ +} + +func R_SYM64(info uint64) uint32 { return uint32(info >> 32) } +func R_TYPE64(info uint64) uint32 { return uint32(info) } +func R_INFO(sym, typ uint32) uint64 { return uint64(sym)<<32 | uint64(typ) } + + +// ELF64 symbol table entries. +type Sym64 struct { + Name uint32 /* String table index of name. */ + Info uint8 /* Type and binding information. */ + Other uint8 /* Reserved (not used). */ + Shndx uint16 /* Section index of symbol. */ + Value uint64 /* Symbol value. */ + Size uint64 /* Size of associated object. */ +} + +const Sym64Size = 24 + +type intName struct { + i uint32 + s string +} + +func stringName(i uint32, names []intName, goSyntax bool) string { + for _, n := range names { + if n.i == i { + if goSyntax { + return "elf." + n.s + } + return n.s + } + } + + // second pass - look for smaller to add with. + // assume sorted already + for j := len(names) - 1; j >= 0; j-- { + n := names[j] + if n.i < i { + s := n.s + if goSyntax { + s = "elf." + s + } + return s + "+" + strconv.Uitoa64(uint64(i-n.i)) + } + } + + return strconv.Uitoa64(uint64(i)) +} + +func flagName(i uint32, names []intName, goSyntax bool) string { + s := "" + for _, n := range names { + if n.i&i == n.i { + if len(s) > 0 { + s += "+" + } + if goSyntax { + s += "elf." + } + s += n.s + i -= n.i + } + } + if len(s) == 0 { + return "0x" + strconv.Uitob64(uint64(i), 16) + } + if i != 0 { + s += "+0x" + strconv.Uitob64(uint64(i), 16) + } + return s +} diff --git a/libgo/go/debug/elf/elf_test.go b/libgo/go/debug/elf/elf_test.go new file mode 100644 index 000000000..67b961b5c --- /dev/null +++ b/libgo/go/debug/elf/elf_test.go @@ -0,0 +1,49 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package elf + +import ( + "fmt" + "testing" +) + +type nameTest struct { + val interface{} + str string +} + +var nameTests = []nameTest{ + {ELFOSABI_LINUX, "ELFOSABI_LINUX"}, + {ET_EXEC, "ET_EXEC"}, + {EM_860, "EM_860"}, + {SHN_LOPROC, "SHN_LOPROC"}, + {SHT_PROGBITS, "SHT_PROGBITS"}, + {SHF_MERGE + SHF_TLS, "SHF_MERGE+SHF_TLS"}, + {PT_LOAD, "PT_LOAD"}, + {PF_W + PF_R + 0x50, "PF_W+PF_R+0x50"}, + {DT_SYMBOLIC, "DT_SYMBOLIC"}, + {DF_BIND_NOW, "DF_BIND_NOW"}, + {NT_FPREGSET, "NT_FPREGSET"}, + {STB_GLOBAL, "STB_GLOBAL"}, + {STT_COMMON, "STT_COMMON"}, + {STV_HIDDEN, "STV_HIDDEN"}, + {R_X86_64_PC32, "R_X86_64_PC32"}, + {R_ALPHA_OP_PUSH, "R_ALPHA_OP_PUSH"}, + {R_ARM_THM_ABS5, "R_ARM_THM_ABS5"}, + {R_386_GOT32, "R_386_GOT32"}, + {R_PPC_GOT16_HI, "R_PPC_GOT16_HI"}, + {R_SPARC_GOT22, "R_SPARC_GOT22"}, + {ET_LOOS + 5, "ET_LOOS+5"}, + {ProgFlag(0x50), "0x50"}, +} + +func TestNames(t *testing.T) { + for i, tt := range nameTests { + s := fmt.Sprint(tt.val) + if s != tt.str { + t.Errorf("#%d: want %q have %q", i, s, tt.str) + } + } +} diff --git a/libgo/go/debug/elf/file.go b/libgo/go/debug/elf/file.go new file mode 100644 index 000000000..e69317a75 --- /dev/null +++ b/libgo/go/debug/elf/file.go @@ -0,0 +1,605 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package elf implements access to ELF object files. +package elf + +import ( + "bytes" + "debug/dwarf" + "encoding/binary" + "fmt" + "io" + "os" +) + +// TODO: error reporting detail + +/* + * Internal ELF representation + */ + +// A FileHeader represents an ELF file header. +type FileHeader struct { + Class Class + Data Data + Version Version + OSABI OSABI + ABIVersion uint8 + ByteOrder binary.ByteOrder + Type Type + Machine Machine +} + +// A File represents an open ELF file. +type File struct { + FileHeader + Sections []*Section + Progs []*Prog + closer io.Closer +} + +// A SectionHeader represents a single ELF section header. +type SectionHeader struct { + Name string + Type SectionType + Flags SectionFlag + Addr uint64 + Offset uint64 + Size uint64 + Link uint32 + Info uint32 + Addralign uint64 + Entsize uint64 +} + +// A Section represents a single section in an ELF file. +type Section struct { + SectionHeader + + // Embed ReaderAt for ReadAt method. + // Do not embed SectionReader directly + // to avoid having Read and Seek. + // If a client wants Read and Seek it must use + // Open() to avoid fighting over the seek offset + // with other clients. + io.ReaderAt + sr *io.SectionReader +} + +// Data reads and returns the contents of the ELF section. +func (s *Section) Data() ([]byte, os.Error) { + dat := make([]byte, s.sr.Size()) + n, err := s.sr.ReadAt(dat, 0) + return dat[0:n], err +} + +// stringTable reads and returns the string table given by the +// specified link value. +func (f *File) stringTable(link uint32) ([]byte, os.Error) { + if link <= 0 || link >= uint32(len(f.Sections)) { + return nil, os.ErrorString("section has invalid string table link") + } + return f.Sections[link].Data() +} + +// Open returns a new ReadSeeker reading the ELF section. +func (s *Section) Open() io.ReadSeeker { return io.NewSectionReader(s.sr, 0, 1<<63-1) } + +// A ProgHeader represents a single ELF program header. +type ProgHeader struct { + Type ProgType + Flags ProgFlag + Vaddr uint64 + Paddr uint64 + Filesz uint64 + Memsz uint64 + Align uint64 +} + +// A Prog represents a single ELF program header in an ELF binary. +type Prog struct { + ProgHeader + + // Embed ReaderAt for ReadAt method. + // Do not embed SectionReader directly + // to avoid having Read and Seek. + // If a client wants Read and Seek it must use + // Open() to avoid fighting over the seek offset + // with other clients. + io.ReaderAt + sr *io.SectionReader +} + +// Open returns a new ReadSeeker reading the ELF program body. +func (p *Prog) Open() io.ReadSeeker { return io.NewSectionReader(p.sr, 0, 1<<63-1) } + +// A Symbol represents an entry in an ELF symbol table section. +type Symbol struct { + Name string + Info, Other byte + Section SectionIndex + Value, Size uint64 +} + +/* + * ELF reader + */ + +type FormatError struct { + off int64 + msg string + val interface{} +} + +func (e *FormatError) String() string { + msg := e.msg + if e.val != nil { + msg += fmt.Sprintf(" '%v' ", e.val) + } + msg += fmt.Sprintf("in record at byte %#x", e.off) + return msg +} + +// Open opens the named file using os.Open and prepares it for use as an ELF binary. +func Open(name string) (*File, os.Error) { + f, err := os.Open(name, os.O_RDONLY, 0) + if err != nil { + return nil, err + } + ff, err := NewFile(f) + if err != nil { + f.Close() + return nil, err + } + ff.closer = f + return ff, nil +} + +// Close closes the File. +// If the File was created using NewFile directly instead of Open, +// Close has no effect. +func (f *File) Close() os.Error { + var err os.Error + if f.closer != nil { + err = f.closer.Close() + f.closer = nil + } + return err +} + +// SectionByType returns the first section in f with the +// given type, or nil if there is no such section. +func (f *File) SectionByType(typ SectionType) *Section { + for _, s := range f.Sections { + if s.Type == typ { + return s + } + } + return nil +} + +// NewFile creates a new File for accessing an ELF binary in an underlying reader. +// The ELF binary is expected to start at position 0 in the ReaderAt. +func NewFile(r io.ReaderAt) (*File, os.Error) { + sr := io.NewSectionReader(r, 0, 1<<63-1) + // Read and decode ELF identifier + var ident [16]uint8 + if _, err := r.ReadAt(ident[0:], 0); err != nil { + return nil, err + } + if ident[0] != '\x7f' || ident[1] != 'E' || ident[2] != 'L' || ident[3] != 'F' { + return nil, &FormatError{0, "bad magic number", ident[0:4]} + } + + f := new(File) + f.Class = Class(ident[EI_CLASS]) + switch f.Class { + case ELFCLASS32: + case ELFCLASS64: + // ok + default: + return nil, &FormatError{0, "unknown ELF class", f.Class} + } + + f.Data = Data(ident[EI_DATA]) + switch f.Data { + case ELFDATA2LSB: + f.ByteOrder = binary.LittleEndian + case ELFDATA2MSB: + f.ByteOrder = binary.BigEndian + default: + return nil, &FormatError{0, "unknown ELF data encoding", f.Data} + } + + f.Version = Version(ident[EI_VERSION]) + if f.Version != EV_CURRENT { + return nil, &FormatError{0, "unknown ELF version", f.Version} + } + + f.OSABI = OSABI(ident[EI_OSABI]) + f.ABIVersion = ident[EI_ABIVERSION] + + // Read ELF file header + var shoff int64 + var shentsize, shnum, shstrndx int + shstrndx = -1 + switch f.Class { + case ELFCLASS32: + hdr := new(Header32) + sr.Seek(0, 0) + if err := binary.Read(sr, f.ByteOrder, hdr); err != nil { + return nil, err + } + f.Type = Type(hdr.Type) + f.Machine = Machine(hdr.Machine) + if v := Version(hdr.Version); v != f.Version { + return nil, &FormatError{0, "mismatched ELF version", v} + } + shoff = int64(hdr.Shoff) + shentsize = int(hdr.Shentsize) + shnum = int(hdr.Shnum) + shstrndx = int(hdr.Shstrndx) + case ELFCLASS64: + hdr := new(Header64) + sr.Seek(0, 0) + if err := binary.Read(sr, f.ByteOrder, hdr); err != nil { + return nil, err + } + f.Type = Type(hdr.Type) + f.Machine = Machine(hdr.Machine) + if v := Version(hdr.Version); v != f.Version { + return nil, &FormatError{0, "mismatched ELF version", v} + } + shoff = int64(hdr.Shoff) + shentsize = int(hdr.Shentsize) + shnum = int(hdr.Shnum) + shstrndx = int(hdr.Shstrndx) + } + if shstrndx < 0 || shstrndx >= shnum { + return nil, &FormatError{0, "invalid ELF shstrndx", shstrndx} + } + + // Read program headers + // TODO + + // Read section headers + f.Sections = make([]*Section, shnum) + names := make([]uint32, shnum) + for i := 0; i < shnum; i++ { + off := shoff + int64(i)*int64(shentsize) + sr.Seek(off, 0) + s := new(Section) + switch f.Class { + case ELFCLASS32: + sh := new(Section32) + if err := binary.Read(sr, f.ByteOrder, sh); err != nil { + return nil, err + } + names[i] = sh.Name + s.SectionHeader = SectionHeader{ + Type: SectionType(sh.Type), + Flags: SectionFlag(sh.Flags), + Addr: uint64(sh.Addr), + Offset: uint64(sh.Off), + Size: uint64(sh.Size), + Link: uint32(sh.Link), + Info: uint32(sh.Info), + Addralign: uint64(sh.Addralign), + Entsize: uint64(sh.Entsize), + } + case ELFCLASS64: + sh := new(Section64) + if err := binary.Read(sr, f.ByteOrder, sh); err != nil { + return nil, err + } + names[i] = sh.Name + s.SectionHeader = SectionHeader{ + Type: SectionType(sh.Type), + Flags: SectionFlag(sh.Flags), + Offset: uint64(sh.Off), + Size: uint64(sh.Size), + Addr: uint64(sh.Addr), + Link: uint32(sh.Link), + Info: uint32(sh.Info), + Addralign: uint64(sh.Addralign), + Entsize: uint64(sh.Entsize), + } + } + s.sr = io.NewSectionReader(r, int64(s.Offset), int64(s.Size)) + s.ReaderAt = s.sr + f.Sections[i] = s + } + + // Load section header string table. + shstrtab, err := f.Sections[shstrndx].Data() + if err != nil { + return nil, err + } + for i, s := range f.Sections { + var ok bool + s.Name, ok = getString(shstrtab, int(names[i])) + if !ok { + return nil, &FormatError{shoff + int64(i*shentsize), "bad section name index", names[i]} + } + } + + return f, nil +} + +// getSymbols returns a slice of Symbols from parsing the symbol table +// with the given type. +func (f *File) getSymbols(typ SectionType) ([]Symbol, os.Error) { + switch f.Class { + case ELFCLASS64: + return f.getSymbols64(typ) + + case ELFCLASS32: + return f.getSymbols32(typ) + } + + return nil, os.ErrorString("not implemented") +} + +func (f *File) getSymbols32(typ SectionType) ([]Symbol, os.Error) { + symtabSection := f.SectionByType(typ) + if symtabSection == nil { + return nil, os.ErrorString("no symbol section") + } + + data, err := symtabSection.Data() + if err != nil { + return nil, os.ErrorString("cannot load symbol section") + } + symtab := bytes.NewBuffer(data) + if symtab.Len()%Sym32Size != 0 { + return nil, os.ErrorString("length of symbol section is not a multiple of SymSize") + } + + strdata, err := f.stringTable(symtabSection.Link) + if err != nil { + return nil, os.ErrorString("cannot load string table section") + } + + // The first entry is all zeros. + var skip [Sym32Size]byte + symtab.Read(skip[0:]) + + symbols := make([]Symbol, symtab.Len()/Sym32Size) + + i := 0 + var sym Sym32 + for symtab.Len() > 0 { + binary.Read(symtab, f.ByteOrder, &sym) + str, _ := getString(strdata, int(sym.Name)) + symbols[i].Name = str + symbols[i].Info = sym.Info + symbols[i].Other = sym.Other + symbols[i].Section = SectionIndex(sym.Shndx) + symbols[i].Value = uint64(sym.Value) + symbols[i].Size = uint64(sym.Size) + i++ + } + + return symbols, nil +} + +func (f *File) getSymbols64(typ SectionType) ([]Symbol, os.Error) { + symtabSection := f.SectionByType(typ) + if symtabSection == nil { + return nil, os.ErrorString("no symbol section") + } + + data, err := symtabSection.Data() + if err != nil { + return nil, os.ErrorString("cannot load symbol section") + } + symtab := bytes.NewBuffer(data) + if symtab.Len()%Sym64Size != 0 { + return nil, os.ErrorString("length of symbol section is not a multiple of Sym64Size") + } + + strdata, err := f.stringTable(symtabSection.Link) + if err != nil { + return nil, os.ErrorString("cannot load string table section") + } + + // The first entry is all zeros. + var skip [Sym64Size]byte + symtab.Read(skip[0:]) + + symbols := make([]Symbol, symtab.Len()/Sym64Size) + + i := 0 + var sym Sym64 + for symtab.Len() > 0 { + binary.Read(symtab, f.ByteOrder, &sym) + str, _ := getString(strdata, int(sym.Name)) + symbols[i].Name = str + symbols[i].Info = sym.Info + symbols[i].Other = sym.Other + symbols[i].Section = SectionIndex(sym.Shndx) + symbols[i].Value = sym.Value + symbols[i].Size = sym.Size + i++ + } + + return symbols, nil +} + +// getString extracts a string from an ELF string table. +func getString(section []byte, start int) (string, bool) { + if start < 0 || start >= len(section) { + return "", false + } + + for end := start; end < len(section); end++ { + if section[end] == 0 { + return string(section[start:end]), true + } + } + return "", false +} + +// Section returns a section with the given name, or nil if no such +// section exists. +func (f *File) Section(name string) *Section { + for _, s := range f.Sections { + if s.Name == name { + return s + } + } + return nil +} + +// applyRelocations applies relocations to dst. rels is a relocations section +// in RELA format. +func (f *File) applyRelocations(dst []byte, rels []byte) os.Error { + if f.Class == ELFCLASS64 && f.Machine == EM_X86_64 { + return f.applyRelocationsAMD64(dst, rels) + } + + return os.ErrorString("not implemented") +} + +func (f *File) applyRelocationsAMD64(dst []byte, rels []byte) os.Error { + if len(rels)%Sym64Size != 0 { + return os.ErrorString("length of relocation section is not a multiple of Sym64Size") + } + + symbols, err := f.getSymbols(SHT_SYMTAB) + if err != nil { + return err + } + + b := bytes.NewBuffer(rels) + var rela Rela64 + + for b.Len() > 0 { + binary.Read(b, f.ByteOrder, &rela) + symNo := rela.Info >> 32 + t := R_X86_64(rela.Info & 0xffff) + + if symNo >= uint64(len(symbols)) { + continue + } + sym := &symbols[symNo] + if SymType(sym.Info&0xf) != STT_SECTION { + // We don't handle non-section relocations for now. + continue + } + + switch t { + case R_X86_64_64: + if rela.Off+8 >= uint64(len(dst)) || rela.Addend < 0 { + continue + } + f.ByteOrder.PutUint64(dst[rela.Off:rela.Off+8], uint64(rela.Addend)) + case R_X86_64_32: + if rela.Off+4 >= uint64(len(dst)) || rela.Addend < 0 { + continue + } + f.ByteOrder.PutUint32(dst[rela.Off:rela.Off+4], uint32(rela.Addend)) + } + } + + return nil +} + +func (f *File) DWARF() (*dwarf.Data, os.Error) { + // There are many other DWARF sections, but these + // are the required ones, and the debug/dwarf package + // does not use the others, so don't bother loading them. + var names = [...]string{"abbrev", "info", "str"} + var dat [len(names)][]byte + for i, name := range names { + name = ".debug_" + name + s := f.Section(name) + if s == nil { + continue + } + b, err := s.Data() + if err != nil && uint64(len(b)) < s.Size { + return nil, err + } + dat[i] = b + } + + // If there's a relocation table for .debug_info, we have to process it + // now otherwise the data in .debug_info is invalid for x86-64 objects. + rela := f.Section(".rela.debug_info") + if rela != nil && rela.Type == SHT_RELA && f.Machine == EM_X86_64 { + data, err := rela.Data() + if err != nil { + return nil, err + } + err = f.applyRelocations(dat[1], data) + if err != nil { + return nil, err + } + } + + abbrev, info, str := dat[0], dat[1], dat[2] + return dwarf.New(abbrev, nil, nil, info, nil, nil, nil, str) +} + +// ImportedSymbols returns the names of all symbols +// referred to by the binary f that are expected to be +// satisfied by other libraries at dynamic load time. +// It does not return weak symbols. +func (f *File) ImportedSymbols() ([]string, os.Error) { + sym, err := f.getSymbols(SHT_DYNSYM) + if err != nil { + return nil, err + } + var all []string + for _, s := range sym { + if ST_BIND(s.Info) == STB_GLOBAL && s.Section == SHN_UNDEF { + all = append(all, s.Name) + } + } + return all, nil +} + +// ImportedLibraries returns the names of all libraries +// referred to by the binary f that are expected to be +// linked with the binary at dynamic link time. +func (f *File) ImportedLibraries() ([]string, os.Error) { + ds := f.SectionByType(SHT_DYNAMIC) + if ds == nil { + // not dynamic, so no libraries + return nil, nil + } + d, err := ds.Data() + if err != nil { + return nil, err + } + str, err := f.stringTable(ds.Link) + if err != nil { + return nil, err + } + var all []string + for len(d) > 0 { + var tag DynTag + var value uint64 + switch f.Class { + case ELFCLASS32: + tag = DynTag(f.ByteOrder.Uint32(d[0:4])) + value = uint64(f.ByteOrder.Uint32(d[4:8])) + d = d[8:] + case ELFCLASS64: + tag = DynTag(f.ByteOrder.Uint64(d[0:8])) + value = f.ByteOrder.Uint64(d[8:16]) + d = d[16:] + } + if tag == DT_NEEDED { + s, ok := getString(str, int(value)) + if ok { + all = append(all, s) + } + } + } + + return all, nil +} diff --git a/libgo/go/debug/elf/file_test.go b/libgo/go/debug/elf/file_test.go new file mode 100644 index 000000000..84068ea12 --- /dev/null +++ b/libgo/go/debug/elf/file_test.go @@ -0,0 +1,180 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package elf + +import ( + "debug/dwarf" + "encoding/binary" + "reflect" + "testing" +) + +type fileTest struct { + file string + hdr FileHeader + sections []SectionHeader +} + +var fileTests = []fileTest{ + { + "testdata/gcc-386-freebsd-exec", + FileHeader{ELFCLASS32, ELFDATA2LSB, EV_CURRENT, ELFOSABI_FREEBSD, 0, binary.LittleEndian, ET_EXEC, EM_386}, + []SectionHeader{ + {"", SHT_NULL, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {".interp", SHT_PROGBITS, SHF_ALLOC, 0x80480d4, 0xd4, 0x15, 0x0, 0x0, 0x1, 0x0}, + {".hash", SHT_HASH, SHF_ALLOC, 0x80480ec, 0xec, 0x90, 0x3, 0x0, 0x4, 0x4}, + {".dynsym", SHT_DYNSYM, SHF_ALLOC, 0x804817c, 0x17c, 0x110, 0x4, 0x1, 0x4, 0x10}, + {".dynstr", SHT_STRTAB, SHF_ALLOC, 0x804828c, 0x28c, 0xbb, 0x0, 0x0, 0x1, 0x0}, + {".rel.plt", SHT_REL, SHF_ALLOC, 0x8048348, 0x348, 0x20, 0x3, 0x7, 0x4, 0x8}, + {".init", SHT_PROGBITS, SHF_ALLOC + SHF_EXECINSTR, 0x8048368, 0x368, 0x11, 0x0, 0x0, 0x4, 0x0}, + {".plt", SHT_PROGBITS, SHF_ALLOC + SHF_EXECINSTR, 0x804837c, 0x37c, 0x50, 0x0, 0x0, 0x4, 0x4}, + {".text", SHT_PROGBITS, SHF_ALLOC + SHF_EXECINSTR, 0x80483cc, 0x3cc, 0x180, 0x0, 0x0, 0x4, 0x0}, + {".fini", SHT_PROGBITS, SHF_ALLOC + SHF_EXECINSTR, 0x804854c, 0x54c, 0xc, 0x0, 0x0, 0x4, 0x0}, + {".rodata", SHT_PROGBITS, SHF_ALLOC, 0x8048558, 0x558, 0xa3, 0x0, 0x0, 0x1, 0x0}, + {".data", SHT_PROGBITS, SHF_WRITE + SHF_ALLOC, 0x80495fc, 0x5fc, 0xc, 0x0, 0x0, 0x4, 0x0}, + {".eh_frame", SHT_PROGBITS, SHF_ALLOC, 0x8049608, 0x608, 0x4, 0x0, 0x0, 0x4, 0x0}, + {".dynamic", SHT_DYNAMIC, SHF_WRITE + SHF_ALLOC, 0x804960c, 0x60c, 0x98, 0x4, 0x0, 0x4, 0x8}, + {".ctors", SHT_PROGBITS, SHF_WRITE + SHF_ALLOC, 0x80496a4, 0x6a4, 0x8, 0x0, 0x0, 0x4, 0x0}, + {".dtors", SHT_PROGBITS, SHF_WRITE + SHF_ALLOC, 0x80496ac, 0x6ac, 0x8, 0x0, 0x0, 0x4, 0x0}, + {".jcr", SHT_PROGBITS, SHF_WRITE + SHF_ALLOC, 0x80496b4, 0x6b4, 0x4, 0x0, 0x0, 0x4, 0x0}, + {".got", SHT_PROGBITS, SHF_WRITE + SHF_ALLOC, 0x80496b8, 0x6b8, 0x1c, 0x0, 0x0, 0x4, 0x4}, + {".bss", SHT_NOBITS, SHF_WRITE + SHF_ALLOC, 0x80496d4, 0x6d4, 0x20, 0x0, 0x0, 0x4, 0x0}, + {".comment", SHT_PROGBITS, 0x0, 0x0, 0x6d4, 0x12d, 0x0, 0x0, 0x1, 0x0}, + {".debug_aranges", SHT_PROGBITS, 0x0, 0x0, 0x801, 0x20, 0x0, 0x0, 0x1, 0x0}, + {".debug_pubnames", SHT_PROGBITS, 0x0, 0x0, 0x821, 0x1b, 0x0, 0x0, 0x1, 0x0}, + {".debug_info", SHT_PROGBITS, 0x0, 0x0, 0x83c, 0x11d, 0x0, 0x0, 0x1, 0x0}, + {".debug_abbrev", SHT_PROGBITS, 0x0, 0x0, 0x959, 0x41, 0x0, 0x0, 0x1, 0x0}, + {".debug_line", SHT_PROGBITS, 0x0, 0x0, 0x99a, 0x35, 0x0, 0x0, 0x1, 0x0}, + {".debug_frame", SHT_PROGBITS, 0x0, 0x0, 0x9d0, 0x30, 0x0, 0x0, 0x4, 0x0}, + {".debug_str", SHT_PROGBITS, 0x0, 0x0, 0xa00, 0xd, 0x0, 0x0, 0x1, 0x0}, + {".shstrtab", SHT_STRTAB, 0x0, 0x0, 0xa0d, 0xf8, 0x0, 0x0, 0x1, 0x0}, + {".symtab", SHT_SYMTAB, 0x0, 0x0, 0xfb8, 0x4b0, 0x1d, 0x38, 0x4, 0x10}, + {".strtab", SHT_STRTAB, 0x0, 0x0, 0x1468, 0x206, 0x0, 0x0, 0x1, 0x0}, + }, + }, + { + "testdata/gcc-amd64-linux-exec", + FileHeader{ELFCLASS64, ELFDATA2LSB, EV_CURRENT, ELFOSABI_NONE, 0, binary.LittleEndian, ET_EXEC, EM_X86_64}, + []SectionHeader{ + {"", SHT_NULL, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {".interp", SHT_PROGBITS, SHF_ALLOC, 0x400200, 0x200, 0x1c, 0x0, 0x0, 0x1, 0x0}, + {".note.ABI-tag", SHT_NOTE, SHF_ALLOC, 0x40021c, 0x21c, 0x20, 0x0, 0x0, 0x4, 0x0}, + {".hash", SHT_HASH, SHF_ALLOC, 0x400240, 0x240, 0x24, 0x5, 0x0, 0x8, 0x4}, + {".gnu.hash", SHT_LOOS + 268435446, SHF_ALLOC, 0x400268, 0x268, 0x1c, 0x5, 0x0, 0x8, 0x0}, + {".dynsym", SHT_DYNSYM, SHF_ALLOC, 0x400288, 0x288, 0x60, 0x6, 0x1, 0x8, 0x18}, + {".dynstr", SHT_STRTAB, SHF_ALLOC, 0x4002e8, 0x2e8, 0x3d, 0x0, 0x0, 0x1, 0x0}, + {".gnu.version", SHT_HIOS, SHF_ALLOC, 0x400326, 0x326, 0x8, 0x5, 0x0, 0x2, 0x2}, + {".gnu.version_r", SHT_LOOS + 268435454, SHF_ALLOC, 0x400330, 0x330, 0x20, 0x6, 0x1, 0x8, 0x0}, + {".rela.dyn", SHT_RELA, SHF_ALLOC, 0x400350, 0x350, 0x18, 0x5, 0x0, 0x8, 0x18}, + {".rela.plt", SHT_RELA, SHF_ALLOC, 0x400368, 0x368, 0x30, 0x5, 0xc, 0x8, 0x18}, + {".init", SHT_PROGBITS, SHF_ALLOC + SHF_EXECINSTR, 0x400398, 0x398, 0x18, 0x0, 0x0, 0x4, 0x0}, + {".plt", SHT_PROGBITS, SHF_ALLOC + SHF_EXECINSTR, 0x4003b0, 0x3b0, 0x30, 0x0, 0x0, 0x4, 0x10}, + {".text", SHT_PROGBITS, SHF_ALLOC + SHF_EXECINSTR, 0x4003e0, 0x3e0, 0x1b4, 0x0, 0x0, 0x10, 0x0}, + {".fini", SHT_PROGBITS, SHF_ALLOC + SHF_EXECINSTR, 0x400594, 0x594, 0xe, 0x0, 0x0, 0x4, 0x0}, + {".rodata", SHT_PROGBITS, SHF_ALLOC, 0x4005a4, 0x5a4, 0x11, 0x0, 0x0, 0x4, 0x0}, + {".eh_frame_hdr", SHT_PROGBITS, SHF_ALLOC, 0x4005b8, 0x5b8, 0x24, 0x0, 0x0, 0x4, 0x0}, + {".eh_frame", SHT_PROGBITS, SHF_ALLOC, 0x4005e0, 0x5e0, 0xa4, 0x0, 0x0, 0x8, 0x0}, + {".ctors", SHT_PROGBITS, SHF_WRITE + SHF_ALLOC, 0x600688, 0x688, 0x10, 0x0, 0x0, 0x8, 0x0}, + {".dtors", SHT_PROGBITS, SHF_WRITE + SHF_ALLOC, 0x600698, 0x698, 0x10, 0x0, 0x0, 0x8, 0x0}, + {".jcr", SHT_PROGBITS, SHF_WRITE + SHF_ALLOC, 0x6006a8, 0x6a8, 0x8, 0x0, 0x0, 0x8, 0x0}, + {".dynamic", SHT_DYNAMIC, SHF_WRITE + SHF_ALLOC, 0x6006b0, 0x6b0, 0x1a0, 0x6, 0x0, 0x8, 0x10}, + {".got", SHT_PROGBITS, SHF_WRITE + SHF_ALLOC, 0x600850, 0x850, 0x8, 0x0, 0x0, 0x8, 0x8}, + {".got.plt", SHT_PROGBITS, SHF_WRITE + SHF_ALLOC, 0x600858, 0x858, 0x28, 0x0, 0x0, 0x8, 0x8}, + {".data", SHT_PROGBITS, SHF_WRITE + SHF_ALLOC, 0x600880, 0x880, 0x18, 0x0, 0x0, 0x8, 0x0}, + {".bss", SHT_NOBITS, SHF_WRITE + SHF_ALLOC, 0x600898, 0x898, 0x8, 0x0, 0x0, 0x4, 0x0}, + {".comment", SHT_PROGBITS, 0x0, 0x0, 0x898, 0x126, 0x0, 0x0, 0x1, 0x0}, + {".debug_aranges", SHT_PROGBITS, 0x0, 0x0, 0x9c0, 0x90, 0x0, 0x0, 0x10, 0x0}, + {".debug_pubnames", SHT_PROGBITS, 0x0, 0x0, 0xa50, 0x25, 0x0, 0x0, 0x1, 0x0}, + {".debug_info", SHT_PROGBITS, 0x0, 0x0, 0xa75, 0x1a7, 0x0, 0x0, 0x1, 0x0}, + {".debug_abbrev", SHT_PROGBITS, 0x0, 0x0, 0xc1c, 0x6f, 0x0, 0x0, 0x1, 0x0}, + {".debug_line", SHT_PROGBITS, 0x0, 0x0, 0xc8b, 0x13f, 0x0, 0x0, 0x1, 0x0}, + {".debug_str", SHT_PROGBITS, SHF_MERGE + SHF_STRINGS, 0x0, 0xdca, 0xb1, 0x0, 0x0, 0x1, 0x1}, + {".debug_ranges", SHT_PROGBITS, 0x0, 0x0, 0xe80, 0x90, 0x0, 0x0, 0x10, 0x0}, + {".shstrtab", SHT_STRTAB, 0x0, 0x0, 0xf10, 0x149, 0x0, 0x0, 0x1, 0x0}, + {".symtab", SHT_SYMTAB, 0x0, 0x0, 0x19a0, 0x6f0, 0x24, 0x39, 0x8, 0x18}, + {".strtab", SHT_STRTAB, 0x0, 0x0, 0x2090, 0x1fc, 0x0, 0x0, 0x1, 0x0}, + }, + }, +} + +func TestOpen(t *testing.T) { + for i := range fileTests { + tt := &fileTests[i] + + f, err := Open(tt.file) + if err != nil { + t.Error(err) + continue + } + if !reflect.DeepEqual(f.FileHeader, tt.hdr) { + t.Errorf("open %s:\n\thave %#v\n\twant %#v\n", tt.file, f.FileHeader, tt.hdr) + continue + } + for i, s := range f.Sections { + if i >= len(tt.sections) { + break + } + sh := &tt.sections[i] + if !reflect.DeepEqual(&s.SectionHeader, sh) { + t.Errorf("open %s, section %d:\n\thave %#v\n\twant %#v\n", tt.file, i, &s.SectionHeader, sh) + } + } + tn := len(tt.sections) + fn := len(f.Sections) + if tn != fn { + t.Errorf("open %s: len(Sections) = %d, want %d", tt.file, fn, tn) + } + } +} + +type relocationTest struct { + file string + firstEntry *dwarf.Entry +} + +var relocationTests = []relocationTest{ + { + "testdata/go-relocation-test-gcc441-x86-64.o", + &dwarf.Entry{Offset: 0xb, Tag: dwarf.TagCompileUnit, Children: true, Field: []dwarf.Field{{Attr: dwarf.AttrProducer, Val: "GNU C 4.4.1"}, {Attr: dwarf.AttrLanguage, Val: int64(1)}, {Attr: dwarf.AttrName, Val: "go-relocation-test.c"}, {Attr: dwarf.AttrCompDir, Val: "/tmp"}, {Attr: dwarf.AttrLowpc, Val: uint64(0x0)}, {Attr: dwarf.AttrHighpc, Val: uint64(0x6)}, {Attr: dwarf.AttrStmtList, Val: int64(0)}}}, + }, + { + "testdata/go-relocation-test-gcc441-x86.o", + &dwarf.Entry{Offset: 0xb, Tag: dwarf.TagCompileUnit, Children: true, Field: []dwarf.Field{{Attr: dwarf.AttrProducer, Val: "GNU C 4.4.1"}, {Attr: dwarf.AttrLanguage, Val: int64(1)}, {Attr: dwarf.AttrName, Val: "t.c"}, {Attr: dwarf.AttrCompDir, Val: "/tmp"}, {Attr: dwarf.AttrLowpc, Val: uint64(0x0)}, {Attr: dwarf.AttrHighpc, Val: uint64(0x5)}, {Attr: dwarf.AttrStmtList, Val: int64(0)}}}, + }, + { + "testdata/go-relocation-test-gcc424-x86-64.o", + &dwarf.Entry{Offset: 0xb, Tag: dwarf.TagCompileUnit, Children: true, Field: []dwarf.Field{{Attr: dwarf.AttrProducer, Val: "GNU C 4.2.4 (Ubuntu 4.2.4-1ubuntu4)"}, {Attr: dwarf.AttrLanguage, Val: int64(1)}, {Attr: dwarf.AttrName, Val: "go-relocation-test-gcc424.c"}, {Attr: dwarf.AttrCompDir, Val: "/tmp"}, {Attr: dwarf.AttrLowpc, Val: uint64(0x0)}, {Attr: dwarf.AttrHighpc, Val: uint64(0x6)}, {Attr: dwarf.AttrStmtList, Val: int64(0)}}}, + }, +} + +func TestDWARFRelocations(t *testing.T) { + for i, test := range relocationTests { + f, err := Open(test.file) + if err != nil { + t.Error(err) + continue + } + dwarf, err := f.DWARF() + if err != nil { + t.Error(err) + continue + } + reader := dwarf.Reader() + // Checking only the first entry is sufficient since it has + // many different strings. If the relocation had failed, all + // the string offsets would be zero and all the strings would + // end up being the same. + firstEntry, err := reader.Next() + if err != nil { + t.Error(err) + continue + } + + if !reflect.DeepEqual(test.firstEntry, firstEntry) { + t.Errorf("#%d: mismatch: got:%#v want:%#v", i, firstEntry, test.firstEntry) + continue + } + } +} diff --git a/libgo/go/debug/elf/testdata/gcc-386-freebsd-exec b/libgo/go/debug/elf/testdata/gcc-386-freebsd-exec new file mode 100755 index 000000000..7af9c58ca Binary files /dev/null and b/libgo/go/debug/elf/testdata/gcc-386-freebsd-exec differ diff --git a/libgo/go/debug/elf/testdata/gcc-amd64-linux-exec b/libgo/go/debug/elf/testdata/gcc-amd64-linux-exec new file mode 100755 index 000000000..c6cb1de28 Binary files /dev/null and b/libgo/go/debug/elf/testdata/gcc-amd64-linux-exec differ diff --git a/libgo/go/debug/elf/testdata/go-relocation-test-gcc424-x86-64.o b/libgo/go/debug/elf/testdata/go-relocation-test-gcc424-x86-64.o new file mode 100644 index 000000000..a7c6d6e56 Binary files /dev/null and b/libgo/go/debug/elf/testdata/go-relocation-test-gcc424-x86-64.o differ diff --git a/libgo/go/debug/elf/testdata/go-relocation-test-gcc441-x86-64.o b/libgo/go/debug/elf/testdata/go-relocation-test-gcc441-x86-64.o new file mode 100644 index 000000000..2d37ab6e6 Binary files /dev/null and b/libgo/go/debug/elf/testdata/go-relocation-test-gcc441-x86-64.o differ diff --git a/libgo/go/debug/elf/testdata/go-relocation-test-gcc441-x86.o b/libgo/go/debug/elf/testdata/go-relocation-test-gcc441-x86.o new file mode 100644 index 000000000..0d59fe303 Binary files /dev/null and b/libgo/go/debug/elf/testdata/go-relocation-test-gcc441-x86.o differ diff --git a/libgo/go/debug/gosym/pclinetest.h b/libgo/go/debug/gosym/pclinetest.h new file mode 100644 index 000000000..a6c40e76c --- /dev/null +++ b/libgo/go/debug/gosym/pclinetest.h @@ -0,0 +1,7 @@ +// Empty include file to generate z symbols + + + + + +// EOF diff --git a/libgo/go/debug/gosym/pclinetest.s b/libgo/go/debug/gosym/pclinetest.s new file mode 100644 index 000000000..519656b63 --- /dev/null +++ b/libgo/go/debug/gosym/pclinetest.s @@ -0,0 +1,89 @@ +TEXT linefrompc(SB),7,$0 // Each byte stores its line delta +BYTE $2; +BYTE $1; +BYTE $1; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; BYTE $0; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; +BYTE $1; +BYTE $1; +BYTE $1; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; BYTE $0; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; +#include "pclinetest.h" +BYTE $2; +#include "pclinetest.h" +BYTE $2; + +TEXT pcfromline(SB),7,$0 // Each record stores its line delta, then n, then n more bytes +BYTE $31; BYTE $0; +BYTE $1; BYTE $1; BYTE $0; +BYTE $1; BYTE $0; + +BYTE $2; BYTE $4; BYTE $0; BYTE $0; BYTE $0; BYTE $0; + + +#include "pclinetest.h" +BYTE $4; BYTE $0; + + +BYTE $3; BYTE $3; BYTE $0; BYTE $0; BYTE $0; +#include "pclinetest.h" + + +BYTE $4; BYTE $3; BYTE $0; BYTE $0; BYTE $0; + +TEXT main(SB),7,$0 + // Prevent GC of our test symbols + CALL linefrompc(SB) + CALL pcfromline(SB) + +// Keep the linker happy +TEXT runtime·morestack(SB),7,$0 + RET + +TEXT runtime·morestack00(SB),7,$0 + RET + +TEXT runtime·morestack10(SB),7,$0 + RET + +TEXT runtime·morestack01(SB),7,$0 + RET + +TEXT runtime·morestack11(SB),7,$0 + RET + +TEXT runtime·morestack8(SB),7,$0 + RET + +TEXT runtime·morestack16(SB),7,$0 + RET + +TEXT runtime·morestack24(SB),7,$0 + RET + +TEXT runtime·morestack32(SB),7,$0 + RET + +TEXT runtime·morestack40(SB),7,$0 + RET + +TEXT runtime·morestack48(SB),7,$0 + RET + +TEXT runtime·morestack8(SB),7,$0 + RET + diff --git a/libgo/go/debug/gosym/pclntab.go b/libgo/go/debug/gosym/pclntab.go new file mode 100644 index 000000000..9d7b0d15f --- /dev/null +++ b/libgo/go/debug/gosym/pclntab.go @@ -0,0 +1,82 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + * Line tables + */ + +package gosym + +import "encoding/binary" + +type LineTable struct { + Data []byte + PC uint64 + Line int +} + +// TODO(rsc): Need to pull in quantum from architecture definition. +const quantum = 1 + +func (t *LineTable) parse(targetPC uint64, targetLine int) (b []byte, pc uint64, line int) { + // The PC/line table can be thought of as a sequence of + // * + // batches. Each update batch results in a (pc, line) pair, + // where line applies to every PC from pc up to but not + // including the pc of the next pair. + // + // Here we process each update individually, which simplifies + // the code, but makes the corner cases more confusing. + b, pc, line = t.Data, t.PC, t.Line + for pc <= targetPC && line != targetLine && len(b) > 0 { + code := b[0] + b = b[1:] + switch { + case code == 0: + if len(b) < 4 { + b = b[0:0] + break + } + val := binary.BigEndian.Uint32(b) + b = b[4:] + line += int(val) + case code <= 64: + line += int(code) + case code <= 128: + line -= int(code - 64) + default: + pc += quantum * uint64(code-128) + continue + } + pc += quantum + } + return b, pc, line +} + +func (t *LineTable) slice(pc uint64) *LineTable { + data, pc, line := t.parse(pc, -1) + return &LineTable{data, pc, line} +} + +func (t *LineTable) PCToLine(pc uint64) int { + _, _, line := t.parse(pc, -1) + return line +} + +func (t *LineTable) LineToPC(line int, maxpc uint64) uint64 { + _, pc, line1 := t.parse(maxpc, line) + if line1 != line { + return 0 + } + // Subtract quantum from PC to account for post-line increment + return pc - quantum +} + +// NewLineTable returns a new PC/line table +// corresponding to the encoded data. +// Text must be the start address of the +// corresponding text segment. +func NewLineTable(data []byte, text uint64) *LineTable { + return &LineTable{data, text, 0} +} diff --git a/libgo/go/debug/gosym/pclntab_test.go b/libgo/go/debug/gosym/pclntab_test.go new file mode 100644 index 000000000..908702173 --- /dev/null +++ b/libgo/go/debug/gosym/pclntab_test.go @@ -0,0 +1,207 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gosym + +import ( + "debug/elf" + "os" + "testing" + "syscall" +) + +func dotest() bool { + // For now, only works on ELF platforms. + return syscall.OS == "linux" && os.Getenv("GOARCH") == "amd64" +} + +func getTable(t *testing.T) *Table { + f, tab := crack(os.Args[0], t) + f.Close() + return tab +} + +func crack(file string, t *testing.T) (*elf.File, *Table) { + // Open self + f, err := elf.Open(file) + if err != nil { + t.Fatal(err) + } + return parse(file, f, t) +} + +func parse(file string, f *elf.File, t *testing.T) (*elf.File, *Table) { + symdat, err := f.Section(".gosymtab").Data() + if err != nil { + f.Close() + t.Fatalf("reading %s gosymtab: %v", file, err) + } + pclndat, err := f.Section(".gopclntab").Data() + if err != nil { + f.Close() + t.Fatalf("reading %s gopclntab: %v", file, err) + } + + pcln := NewLineTable(pclndat, f.Section(".text").Addr) + tab, err := NewTable(symdat, pcln) + if err != nil { + f.Close() + t.Fatalf("parsing %s gosymtab: %v", file, err) + } + + return f, tab +} + +var goarch = os.Getenv("O") + +func TestLineFromAline(t *testing.T) { + if !dotest() { + return + } + + tab := getTable(t) + + // Find the sym package + pkg := tab.LookupFunc("debug/gosym.TestLineFromAline").Obj + if pkg == nil { + t.Fatalf("nil pkg") + } + + // Walk every absolute line and ensure that we hit every + // source line monotonically + lastline := make(map[string]int) + final := -1 + for i := 0; i < 10000; i++ { + path, line := pkg.lineFromAline(i) + // Check for end of object + if path == "" { + if final == -1 { + final = i - 1 + } + continue + } else if final != -1 { + t.Fatalf("reached end of package at absolute line %d, but absolute line %d mapped to %s:%d", final, i, path, line) + } + // It's okay to see files multiple times (e.g., sys.a) + if line == 1 { + lastline[path] = 1 + continue + } + // Check that the is the next line in path + ll, ok := lastline[path] + if !ok { + t.Errorf("file %s starts on line %d", path, line) + } else if line != ll+1 { + t.Errorf("expected next line of file %s to be %d, got %d", path, ll+1, line) + } + lastline[path] = line + } + if final == -1 { + t.Errorf("never reached end of object") + } +} + +func TestLineAline(t *testing.T) { + if !dotest() { + return + } + + tab := getTable(t) + + for _, o := range tab.Files { + // A source file can appear multiple times in a + // object. alineFromLine will always return alines in + // the first file, so track which lines we've seen. + found := make(map[string]int) + for i := 0; i < 1000; i++ { + path, line := o.lineFromAline(i) + if path == "" { + break + } + + // cgo files are full of 'Z' symbols, which we don't handle + if len(path) > 4 && path[len(path)-4:] == ".cgo" { + continue + } + + if minline, ok := found[path]; path != "" && ok { + if minline >= line { + // We've already covered this file + continue + } + } + found[path] = line + + a, err := o.alineFromLine(path, line) + if err != nil { + t.Errorf("absolute line %d in object %s maps to %s:%d, but mapping that back gives error %s", i, o.Paths[0].Name, path, line, err) + } else if a != i { + t.Errorf("absolute line %d in object %s maps to %s:%d, which maps back to absolute line %d\n", i, o.Paths[0].Name, path, line, a) + } + } + } +} + +// gotest: if [ "$(uname)-$(uname -m)" = Linux-x86_64 -a "$GOARCH" = amd64 ]; then +// gotest: mkdir -p _test && $AS pclinetest.s && $LD -E main -o _test/pclinetest pclinetest.$O +// gotest: fi +func TestPCLine(t *testing.T) { + if !dotest() { + return + } + + f, tab := crack("_test/pclinetest", t) + text := f.Section(".text") + textdat, err := text.Data() + if err != nil { + t.Fatalf("reading .text: %v", err) + } + + // Test PCToLine + sym := tab.LookupFunc("linefrompc") + wantLine := 0 + for pc := sym.Entry; pc < sym.End; pc++ { + file, line, fn := tab.PCToLine(pc) + off := pc - text.Addr // TODO(rsc): should not need off; bug in 8g + wantLine += int(textdat[off]) + if fn == nil { + t.Errorf("failed to get line of PC %#x", pc) + } else if len(file) < 12 || file[len(file)-12:] != "pclinetest.s" || line != wantLine || fn != sym { + t.Errorf("expected %s:%d (%s) at PC %#x, got %s:%d (%s)", "pclinetest.s", wantLine, sym.Name, pc, file, line, fn.Name) + } + } + + // Test LineToPC + sym = tab.LookupFunc("pcfromline") + lookupline := -1 + wantLine = 0 + off := uint64(0) // TODO(rsc): should not need off; bug in 8g + for pc := sym.Value; pc < sym.End; pc += 2 + uint64(textdat[off]) { + file, line, fn := tab.PCToLine(pc) + off = pc - text.Addr + wantLine += int(textdat[off]) + if line != wantLine { + t.Errorf("expected line %d at PC %#x in pcfromline, got %d", wantLine, pc, line) + off = pc + 1 - text.Addr + continue + } + if lookupline == -1 { + lookupline = line + } + for ; lookupline <= line; lookupline++ { + pc2, fn2, err := tab.LineToPC(file, lookupline) + if lookupline != line { + // Should be nothing on this line + if err == nil { + t.Errorf("expected no PC at line %d, got %#x (%s)", lookupline, pc2, fn2.Name) + } + } else if err != nil { + t.Errorf("failed to get PC of line %d: %s", lookupline, err) + } else if pc != pc2 { + t.Errorf("expected PC %#x (%s) at line %d, got PC %#x (%s)", pc, fn.Name, line, pc2, fn2.Name) + } + } + off = pc + 1 - text.Addr + } +} diff --git a/libgo/go/debug/gosym/symtab.go b/libgo/go/debug/gosym/symtab.go new file mode 100644 index 000000000..dea460d71 --- /dev/null +++ b/libgo/go/debug/gosym/symtab.go @@ -0,0 +1,548 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package gosym implements access to the Go symbol +// and line number tables embedded in Go binaries generated +// by the gc compilers. +package gosym + +// The table format is a variant of the format used in Plan 9's a.out +// format, documented at http://plan9.bell-labs.com/magic/man2html/6/a.out. +// The best reference for the differences between the Plan 9 format +// and the Go format is the runtime source, specifically ../../runtime/symtab.c. + +import ( + "encoding/binary" + "fmt" + "os" + "strconv" + "strings" +) + +/* + * Symbols + */ + +// A Sym represents a single symbol table entry. +type Sym struct { + Value uint64 + Type byte + Name string + GoType uint64 + // If this symbol if a function symbol, the corresponding Func + Func *Func +} + +// Static returns whether this symbol is static (not visible outside its file). +func (s *Sym) Static() bool { return s.Type >= 'a' } + +// PackageName returns the package part of the symbol name, +// or the empty string if there is none. +func (s *Sym) PackageName() string { + if i := strings.Index(s.Name, "."); i != -1 { + return s.Name[0:i] + } + return "" +} + +// ReceiverName returns the receiver type name of this symbol, +// or the empty string if there is none. +func (s *Sym) ReceiverName() string { + l := strings.Index(s.Name, ".") + r := strings.LastIndex(s.Name, ".") + if l == -1 || r == -1 || l == r { + return "" + } + return s.Name[l+1 : r] +} + +// BaseName returns the symbol name without the package or receiver name. +func (s *Sym) BaseName() string { + if i := strings.LastIndex(s.Name, "."); i != -1 { + return s.Name[i+1:] + } + return s.Name +} + +// A Func collects information about a single function. +type Func struct { + Entry uint64 + *Sym + End uint64 + Params []*Sym + Locals []*Sym + FrameSize int + LineTable *LineTable + Obj *Obj +} + +// An Obj represents a single object file. +type Obj struct { + Funcs []Func + Paths []Sym +} + +/* + * Symbol tables + */ + +// Table represents a Go symbol table. It stores all of the +// symbols decoded from the program and provides methods to translate +// between symbols, names, and addresses. +type Table struct { + Syms []Sym + Funcs []Func + Files map[string]*Obj + Objs []Obj + // textEnd uint64; +} + +type sym struct { + value uint32 + gotype uint32 + typ byte + name []byte +} + +func walksymtab(data []byte, fn func(sym) os.Error) os.Error { + var s sym + p := data + for len(p) >= 6 { + s.value = binary.BigEndian.Uint32(p[0:4]) + typ := p[4] + if typ&0x80 == 0 { + return &DecodingError{len(data) - len(p) + 4, "bad symbol type", typ} + } + typ &^= 0x80 + s.typ = typ + p = p[5:] + var i int + var nnul int + for i = 0; i < len(p); i++ { + if p[i] == 0 { + nnul = 1 + break + } + } + switch typ { + case 'z', 'Z': + p = p[i+nnul:] + for i = 0; i+2 <= len(p); i += 2 { + if p[i] == 0 && p[i+1] == 0 { + nnul = 2 + break + } + } + } + if i+nnul+4 > len(p) { + return &DecodingError{len(data), "unexpected EOF", nil} + } + s.name = p[0:i] + i += nnul + s.gotype = binary.BigEndian.Uint32(p[i : i+4]) + p = p[i+4:] + fn(s) + } + return nil +} + +// NewTable decodes the Go symbol table in data, +// returning an in-memory representation. +func NewTable(symtab []byte, pcln *LineTable) (*Table, os.Error) { + var n int + err := walksymtab(symtab, func(s sym) os.Error { + n++ + return nil + }) + if err != nil { + return nil, err + } + + var t Table + fname := make(map[uint16]string) + t.Syms = make([]Sym, 0, n) + nf := 0 + nz := 0 + lasttyp := uint8(0) + err = walksymtab(symtab, func(s sym) os.Error { + n := len(t.Syms) + t.Syms = t.Syms[0 : n+1] + ts := &t.Syms[n] + ts.Type = s.typ + ts.Value = uint64(s.value) + ts.GoType = uint64(s.gotype) + switch s.typ { + default: + // rewrite name to use . instead of · (c2 b7) + w := 0 + b := s.name + for i := 0; i < len(b); i++ { + if b[i] == 0xc2 && i+1 < len(b) && b[i+1] == 0xb7 { + i++ + b[i] = '.' + } + b[w] = b[i] + w++ + } + ts.Name = string(s.name[0:w]) + case 'z', 'Z': + if lasttyp != 'z' && lasttyp != 'Z' { + nz++ + } + for i := 0; i < len(s.name); i += 2 { + eltIdx := binary.BigEndian.Uint16(s.name[i : i+2]) + elt, ok := fname[eltIdx] + if !ok { + return &DecodingError{-1, "bad filename code", eltIdx} + } + if n := len(ts.Name); n > 0 && ts.Name[n-1] != '/' { + ts.Name += "/" + } + ts.Name += elt + } + } + switch s.typ { + case 'T', 't', 'L', 'l': + nf++ + case 'f': + fname[uint16(s.value)] = ts.Name + } + lasttyp = s.typ + return nil + }) + if err != nil { + return nil, err + } + + t.Funcs = make([]Func, 0, nf) + t.Objs = make([]Obj, 0, nz) + t.Files = make(map[string]*Obj) + + // Count text symbols and attach frame sizes, parameters, and + // locals to them. Also, find object file boundaries. + var obj *Obj + lastf := 0 + for i := 0; i < len(t.Syms); i++ { + sym := &t.Syms[i] + switch sym.Type { + case 'Z', 'z': // path symbol + // Finish the current object + if obj != nil { + obj.Funcs = t.Funcs[lastf:] + } + lastf = len(t.Funcs) + + // Start new object + n := len(t.Objs) + t.Objs = t.Objs[0 : n+1] + obj = &t.Objs[n] + + // Count & copy path symbols + var end int + for end = i + 1; end < len(t.Syms); end++ { + if c := t.Syms[end].Type; c != 'Z' && c != 'z' { + break + } + } + obj.Paths = t.Syms[i:end] + i = end - 1 // loop will i++ + + // Record file names + depth := 0 + for j := range obj.Paths { + s := &obj.Paths[j] + if s.Name == "" { + depth-- + } else { + if depth == 0 { + t.Files[s.Name] = obj + } + depth++ + } + } + + case 'T', 't', 'L', 'l': // text symbol + if n := len(t.Funcs); n > 0 { + t.Funcs[n-1].End = sym.Value + } + if sym.Name == "etext" { + continue + } + + // Count parameter and local (auto) syms + var np, na int + var end int + countloop: + for end = i + 1; end < len(t.Syms); end++ { + switch t.Syms[end].Type { + case 'T', 't', 'L', 'l', 'Z', 'z': + break countloop + case 'p': + np++ + case 'a': + na++ + } + } + + // Fill in the function symbol + n := len(t.Funcs) + t.Funcs = t.Funcs[0 : n+1] + fn := &t.Funcs[n] + sym.Func = fn + fn.Params = make([]*Sym, 0, np) + fn.Locals = make([]*Sym, 0, na) + fn.Sym = sym + fn.Entry = sym.Value + fn.Obj = obj + if pcln != nil { + fn.LineTable = pcln.slice(fn.Entry) + pcln = fn.LineTable + } + for j := i; j < end; j++ { + s := &t.Syms[j] + switch s.Type { + case 'm': + fn.FrameSize = int(s.Value) + case 'p': + n := len(fn.Params) + fn.Params = fn.Params[0 : n+1] + fn.Params[n] = s + case 'a': + n := len(fn.Locals) + fn.Locals = fn.Locals[0 : n+1] + fn.Locals[n] = s + } + } + i = end - 1 // loop will i++ + } + } + if obj != nil { + obj.Funcs = t.Funcs[lastf:] + } + return &t, nil +} + +// PCToFunc returns the function containing the program counter pc, +// or nil if there is no such function. +func (t *Table) PCToFunc(pc uint64) *Func { + funcs := t.Funcs + for len(funcs) > 0 { + m := len(funcs) / 2 + fn := &funcs[m] + switch { + case pc < fn.Entry: + funcs = funcs[0:m] + case fn.Entry <= pc && pc < fn.End: + return fn + default: + funcs = funcs[m+1:] + } + } + return nil +} + +// PCToLine looks up line number information for a program counter. +// If there is no information, it returns fn == nil. +func (t *Table) PCToLine(pc uint64) (file string, line int, fn *Func) { + if fn = t.PCToFunc(pc); fn == nil { + return + } + file, line = fn.Obj.lineFromAline(fn.LineTable.PCToLine(pc)) + return +} + +// LineToPC looks up the first program counter on the given line in +// the named file. Returns UnknownPathError or UnknownLineError if +// there is an error looking up this line. +func (t *Table) LineToPC(file string, line int) (pc uint64, fn *Func, err os.Error) { + obj, ok := t.Files[file] + if !ok { + return 0, nil, UnknownFileError(file) + } + abs, err := obj.alineFromLine(file, line) + if err != nil { + return + } + for i := range obj.Funcs { + f := &obj.Funcs[i] + pc := f.LineTable.LineToPC(abs, f.End) + if pc != 0 { + return pc, f, nil + } + } + return 0, nil, &UnknownLineError{file, line} +} + +// LookupSym returns the text, data, or bss symbol with the given name, +// or nil if no such symbol is found. +func (t *Table) LookupSym(name string) *Sym { + // TODO(austin) Maybe make a map + for i := range t.Syms { + s := &t.Syms[i] + switch s.Type { + case 'T', 't', 'L', 'l', 'D', 'd', 'B', 'b': + if s.Name == name { + return s + } + } + } + return nil +} + +// LookupFunc returns the text, data, or bss symbol with the given name, +// or nil if no such symbol is found. +func (t *Table) LookupFunc(name string) *Func { + for i := range t.Funcs { + f := &t.Funcs[i] + if f.Sym.Name == name { + return f + } + } + return nil +} + +// SymByAddr returns the text, data, or bss symbol starting at the given address. +// TODO(rsc): Allow lookup by any address within the symbol. +func (t *Table) SymByAddr(addr uint64) *Sym { + // TODO(austin) Maybe make a map + for i := range t.Syms { + s := &t.Syms[i] + switch s.Type { + case 'T', 't', 'L', 'l', 'D', 'd', 'B', 'b': + if s.Value == addr { + return s + } + } + } + return nil +} + +/* + * Object files + */ + +func (o *Obj) lineFromAline(aline int) (string, int) { + type stackEnt struct { + path string + start int + offset int + prev *stackEnt + } + + noPath := &stackEnt{"", 0, 0, nil} + tos := noPath + + // TODO(austin) I have no idea how 'Z' symbols work, except + // that they pop the stack. +pathloop: + for _, s := range o.Paths { + val := int(s.Value) + switch { + case val > aline: + break pathloop + + case val == 1: + // Start a new stack + tos = &stackEnt{s.Name, val, 0, noPath} + + case s.Name == "": + // Pop + if tos == noPath { + return "", 0 + } + tos.prev.offset += val - tos.start + tos = tos.prev + + default: + // Push + tos = &stackEnt{s.Name, val, 0, tos} + } + } + + if tos == noPath { + return "", 0 + } + return tos.path, aline - tos.start - tos.offset + 1 +} + +func (o *Obj) alineFromLine(path string, line int) (int, os.Error) { + if line < 1 { + return 0, &UnknownLineError{path, line} + } + + for i, s := range o.Paths { + // Find this path + if s.Name != path { + continue + } + + // Find this line at this stack level + depth := 0 + var incstart int + line += int(s.Value) + pathloop: + for _, s := range o.Paths[i:] { + val := int(s.Value) + switch { + case depth == 1 && val >= line: + return line - 1, nil + + case s.Name == "": + depth-- + if depth == 0 { + break pathloop + } else if depth == 1 { + line += val - incstart + } + + default: + if depth == 1 { + incstart = val + } + depth++ + } + } + return 0, &UnknownLineError{path, line} + } + return 0, UnknownFileError(path) +} + +/* + * Errors + */ + +// UnknownFileError represents a failure to find the specific file in +// the symbol table. +type UnknownFileError string + +func (e UnknownFileError) String() string { return "unknown file: " + string(e) } + +// UnknownLineError represents a failure to map a line to a program +// counter, either because the line is beyond the bounds of the file +// or because there is no code on the given line. +type UnknownLineError struct { + File string + Line int +} + +func (e *UnknownLineError) String() string { + return "no code at " + e.File + ":" + strconv.Itoa(e.Line) +} + +// DecodingError represents an error during the decoding of +// the symbol table. +type DecodingError struct { + off int + msg string + val interface{} +} + +func (e *DecodingError) String() string { + msg := e.msg + if e.val != nil { + msg += fmt.Sprintf(" '%v'", e.val) + } + msg += fmt.Sprintf(" at byte %#x", e.off) + return msg +} diff --git a/libgo/go/debug/macho/file.go b/libgo/go/debug/macho/file.go new file mode 100644 index 000000000..fd8da9449 --- /dev/null +++ b/libgo/go/debug/macho/file.go @@ -0,0 +1,517 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package macho implements access to Mach-O object files, as defined by +// http://developer.apple.com/mac/library/documentation/DeveloperTools/Conceptual/MachORuntime/Reference/reference.html. +package macho + +// High level access to low level data structures. + +import ( + "bytes" + "debug/dwarf" + "encoding/binary" + "fmt" + "io" + "os" +) + +// A File represents an open Mach-O file. +type File struct { + FileHeader + ByteOrder binary.ByteOrder + Loads []Load + Sections []*Section + + Symtab *Symtab + Dysymtab *Dysymtab + + closer io.Closer +} + +// A Load represents any Mach-O load command. +type Load interface { + Raw() []byte +} + +// A LoadBytes is the uninterpreted bytes of a Mach-O load command. +type LoadBytes []byte + +func (b LoadBytes) Raw() []byte { return b } + +// A SegmentHeader is the header for a Mach-O 32-bit or 64-bit load segment command. +type SegmentHeader struct { + Cmd LoadCmd + Len uint32 + Name string + Addr uint64 + Memsz uint64 + Offset uint64 + Filesz uint64 + Maxprot uint32 + Prot uint32 + Nsect uint32 + Flag uint32 +} + +// A Segment represents a Mach-O 32-bit or 64-bit load segment command. +type Segment struct { + LoadBytes + SegmentHeader + + // Embed ReaderAt for ReadAt method. + // Do not embed SectionReader directly + // to avoid having Read and Seek. + // If a client wants Read and Seek it must use + // Open() to avoid fighting over the seek offset + // with other clients. + io.ReaderAt + sr *io.SectionReader +} + +// Data reads and returns the contents of the segment. +func (s *Segment) Data() ([]byte, os.Error) { + dat := make([]byte, s.sr.Size()) + n, err := s.sr.ReadAt(dat, 0) + return dat[0:n], err +} + +// Open returns a new ReadSeeker reading the segment. +func (s *Segment) Open() io.ReadSeeker { return io.NewSectionReader(s.sr, 0, 1<<63-1) } + +type SectionHeader struct { + Name string + Seg string + Addr uint64 + Size uint64 + Offset uint32 + Align uint32 + Reloff uint32 + Nreloc uint32 + Flags uint32 +} + +type Section struct { + SectionHeader + + // Embed ReaderAt for ReadAt method. + // Do not embed SectionReader directly + // to avoid having Read and Seek. + // If a client wants Read and Seek it must use + // Open() to avoid fighting over the seek offset + // with other clients. + io.ReaderAt + sr *io.SectionReader +} + +// Data reads and returns the contents of the Mach-O section. +func (s *Section) Data() ([]byte, os.Error) { + dat := make([]byte, s.sr.Size()) + n, err := s.sr.ReadAt(dat, 0) + return dat[0:n], err +} + +// Open returns a new ReadSeeker reading the Mach-O section. +func (s *Section) Open() io.ReadSeeker { return io.NewSectionReader(s.sr, 0, 1<<63-1) } + +// A Dylib represents a Mach-O load dynamic library command. +type Dylib struct { + LoadBytes + Name string + Time uint32 + CurrentVersion uint32 + CompatVersion uint32 +} + +// A Symtab represents a Mach-O symbol table command. +type Symtab struct { + LoadBytes + SymtabCmd + Syms []Symbol +} + +// A Dysymtab represents a Mach-O dynamic symbol table command. +type Dysymtab struct { + LoadBytes + DysymtabCmd + IndirectSyms []uint32 // indices into Symtab.Syms +} + +/* + * Mach-O reader + */ + +type FormatError struct { + off int64 + msg string + val interface{} +} + +func (e *FormatError) String() string { + msg := e.msg + if e.val != nil { + msg += fmt.Sprintf(" '%v'", e.val) + } + msg += fmt.Sprintf(" in record at byte %#x", e.off) + return msg +} + +// Open opens the named file using os.Open and prepares it for use as a Mach-O binary. +func Open(name string) (*File, os.Error) { + f, err := os.Open(name, os.O_RDONLY, 0) + if err != nil { + return nil, err + } + ff, err := NewFile(f) + if err != nil { + f.Close() + return nil, err + } + ff.closer = f + return ff, nil +} + +// Close closes the File. +// If the File was created using NewFile directly instead of Open, +// Close has no effect. +func (f *File) Close() os.Error { + var err os.Error + if f.closer != nil { + err = f.closer.Close() + f.closer = nil + } + return err +} + +// NewFile creates a new File for acecssing a Mach-O binary in an underlying reader. +// The Mach-O binary is expected to start at position 0 in the ReaderAt. +func NewFile(r io.ReaderAt) (*File, os.Error) { + f := new(File) + sr := io.NewSectionReader(r, 0, 1<<63-1) + + // Read and decode Mach magic to determine byte order, size. + // Magic32 and Magic64 differ only in the bottom bit. + var ident [4]byte + if _, err := r.ReadAt(ident[0:], 0); err != nil { + return nil, err + } + be := binary.BigEndian.Uint32(ident[0:]) + le := binary.LittleEndian.Uint32(ident[0:]) + switch Magic32 &^ 1 { + case be &^ 1: + f.ByteOrder = binary.BigEndian + f.Magic = be + case le &^ 1: + f.ByteOrder = binary.LittleEndian + f.Magic = le + default: + return nil, &FormatError{0, "invalid magic number", nil} + } + + // Read entire file header. + if err := binary.Read(sr, f.ByteOrder, &f.FileHeader); err != nil { + return nil, err + } + + // Then load commands. + offset := int64(fileHeaderSize32) + if f.Magic == Magic64 { + offset = fileHeaderSize64 + } + dat := make([]byte, f.Cmdsz) + if _, err := r.ReadAt(dat, offset); err != nil { + return nil, err + } + f.Loads = make([]Load, f.Ncmd) + bo := f.ByteOrder + for i := range f.Loads { + // Each load command begins with uint32 command and length. + if len(dat) < 8 { + return nil, &FormatError{offset, "command block too small", nil} + } + cmd, siz := LoadCmd(bo.Uint32(dat[0:4])), bo.Uint32(dat[4:8]) + if siz < 8 || siz > uint32(len(dat)) { + return nil, &FormatError{offset, "invalid command block size", nil} + } + var cmddat []byte + cmddat, dat = dat[0:siz], dat[siz:] + offset += int64(siz) + var s *Segment + switch cmd { + default: + f.Loads[i] = LoadBytes(cmddat) + + case LoadCmdDylib: + var hdr DylibCmd + b := bytes.NewBuffer(cmddat) + if err := binary.Read(b, bo, &hdr); err != nil { + return nil, err + } + l := new(Dylib) + if hdr.Name >= uint32(len(cmddat)) { + return nil, &FormatError{offset, "invalid name in dynamic library command", hdr.Name} + } + l.Name = cstring(cmddat[hdr.Name:]) + l.Time = hdr.Time + l.CurrentVersion = hdr.CurrentVersion + l.CompatVersion = hdr.CompatVersion + l.LoadBytes = LoadBytes(cmddat) + f.Loads[i] = l + + case LoadCmdSymtab: + var hdr SymtabCmd + b := bytes.NewBuffer(cmddat) + if err := binary.Read(b, bo, &hdr); err != nil { + return nil, err + } + strtab := make([]byte, hdr.Strsize) + if _, err := r.ReadAt(strtab, int64(hdr.Stroff)); err != nil { + return nil, err + } + var symsz int + if f.Magic == Magic64 { + symsz = 16 + } else { + symsz = 12 + } + symdat := make([]byte, int(hdr.Nsyms)*symsz) + if _, err := r.ReadAt(symdat, int64(hdr.Symoff)); err != nil { + return nil, err + } + st, err := f.parseSymtab(symdat, strtab, cmddat, &hdr, offset) + if err != nil { + return nil, err + } + f.Loads[i] = st + f.Symtab = st + + case LoadCmdDysymtab: + var hdr DysymtabCmd + b := bytes.NewBuffer(cmddat) + if err := binary.Read(b, bo, &hdr); err != nil { + return nil, err + } + dat := make([]byte, hdr.Nindirectsyms*4) + if _, err := r.ReadAt(dat, int64(hdr.Indirectsymoff)); err != nil { + return nil, err + } + x := make([]uint32, hdr.Nindirectsyms) + if err := binary.Read(bytes.NewBuffer(dat), bo, x); err != nil { + return nil, err + } + st := new(Dysymtab) + st.LoadBytes = LoadBytes(cmddat) + st.DysymtabCmd = hdr + st.IndirectSyms = x + f.Loads[i] = st + f.Dysymtab = st + + case LoadCmdSegment: + var seg32 Segment32 + b := bytes.NewBuffer(cmddat) + if err := binary.Read(b, bo, &seg32); err != nil { + return nil, err + } + s = new(Segment) + s.LoadBytes = cmddat + s.Cmd = cmd + s.Len = siz + s.Name = cstring(seg32.Name[0:]) + s.Addr = uint64(seg32.Addr) + s.Memsz = uint64(seg32.Memsz) + s.Offset = uint64(seg32.Offset) + s.Filesz = uint64(seg32.Filesz) + s.Maxprot = seg32.Maxprot + s.Prot = seg32.Prot + s.Nsect = seg32.Nsect + s.Flag = seg32.Flag + f.Loads[i] = s + for i := 0; i < int(s.Nsect); i++ { + var sh32 Section32 + if err := binary.Read(b, bo, &sh32); err != nil { + return nil, err + } + sh := new(Section) + sh.Name = cstring(sh32.Name[0:]) + sh.Seg = cstring(sh32.Seg[0:]) + sh.Addr = uint64(sh32.Addr) + sh.Size = uint64(sh32.Size) + sh.Offset = sh32.Offset + sh.Align = sh32.Align + sh.Reloff = sh32.Reloff + sh.Nreloc = sh32.Nreloc + sh.Flags = sh32.Flags + f.pushSection(sh, r) + } + + case LoadCmdSegment64: + var seg64 Segment64 + b := bytes.NewBuffer(cmddat) + if err := binary.Read(b, bo, &seg64); err != nil { + return nil, err + } + s = new(Segment) + s.LoadBytes = cmddat + s.Cmd = cmd + s.Len = siz + s.Name = cstring(seg64.Name[0:]) + s.Addr = seg64.Addr + s.Memsz = seg64.Memsz + s.Offset = seg64.Offset + s.Filesz = seg64.Filesz + s.Maxprot = seg64.Maxprot + s.Prot = seg64.Prot + s.Nsect = seg64.Nsect + s.Flag = seg64.Flag + f.Loads[i] = s + for i := 0; i < int(s.Nsect); i++ { + var sh64 Section64 + if err := binary.Read(b, bo, &sh64); err != nil { + return nil, err + } + sh := new(Section) + sh.Name = cstring(sh64.Name[0:]) + sh.Seg = cstring(sh64.Seg[0:]) + sh.Addr = sh64.Addr + sh.Size = sh64.Size + sh.Offset = sh64.Offset + sh.Align = sh64.Align + sh.Reloff = sh64.Reloff + sh.Nreloc = sh64.Nreloc + sh.Flags = sh64.Flags + f.pushSection(sh, r) + } + } + if s != nil { + s.sr = io.NewSectionReader(r, int64(s.Offset), int64(s.Filesz)) + s.ReaderAt = s.sr + } + } + return f, nil +} + +func (f *File) parseSymtab(symdat, strtab, cmddat []byte, hdr *SymtabCmd, offset int64) (*Symtab, os.Error) { + bo := f.ByteOrder + symtab := make([]Symbol, hdr.Nsyms) + b := bytes.NewBuffer(symdat) + for i := range symtab { + var n Nlist64 + if f.Magic == Magic64 { + if err := binary.Read(b, bo, &n); err != nil { + return nil, err + } + } else { + var n32 Nlist32 + if err := binary.Read(b, bo, &n32); err != nil { + return nil, err + } + n.Name = n32.Name + n.Type = n32.Type + n.Sect = n32.Sect + n.Desc = n32.Desc + n.Value = uint64(n32.Value) + } + sym := &symtab[i] + if n.Name >= uint32(len(strtab)) { + return nil, &FormatError{offset, "invalid name in symbol table", n.Name} + } + sym.Name = cstring(strtab[n.Name:]) + sym.Type = n.Type + sym.Sect = n.Sect + sym.Desc = n.Desc + sym.Value = n.Value + } + st := new(Symtab) + st.LoadBytes = LoadBytes(cmddat) + st.Syms = symtab + return st, nil +} + +func (f *File) pushSection(sh *Section, r io.ReaderAt) { + f.Sections = append(f.Sections, sh) + sh.sr = io.NewSectionReader(r, int64(sh.Offset), int64(sh.Size)) + sh.ReaderAt = sh.sr +} + +func cstring(b []byte) string { + var i int + for i = 0; i < len(b) && b[i] != 0; i++ { + } + return string(b[0:i]) +} + +// Segment returns the first Segment with the given name, or nil if no such segment exists. +func (f *File) Segment(name string) *Segment { + for _, l := range f.Loads { + if s, ok := l.(*Segment); ok && s.Name == name { + return s + } + } + return nil +} + +// Section returns the first section with the given name, or nil if no such +// section exists. +func (f *File) Section(name string) *Section { + for _, s := range f.Sections { + if s.Name == name { + return s + } + } + return nil +} + +// DWARF returns the DWARF debug information for the Mach-O file. +func (f *File) DWARF() (*dwarf.Data, os.Error) { + // There are many other DWARF sections, but these + // are the required ones, and the debug/dwarf package + // does not use the others, so don't bother loading them. + var names = [...]string{"abbrev", "info", "str"} + var dat [len(names)][]byte + for i, name := range names { + name = "__debug_" + name + s := f.Section(name) + if s == nil { + return nil, os.NewError("missing Mach-O section " + name) + } + b, err := s.Data() + if err != nil && uint64(len(b)) < s.Size { + return nil, err + } + dat[i] = b + } + + abbrev, info, str := dat[0], dat[1], dat[2] + return dwarf.New(abbrev, nil, nil, info, nil, nil, nil, str) +} + +// ImportedSymbols returns the names of all symbols +// referred to by the binary f that are expected to be +// satisfied by other libraries at dynamic load time. +func (f *File) ImportedSymbols() ([]string, os.Error) { + if f.Dysymtab == nil || f.Symtab == nil { + return nil, &FormatError{0, "missing symbol table", nil} + } + + st := f.Symtab + dt := f.Dysymtab + var all []string + for _, s := range st.Syms[dt.Iundefsym : dt.Iundefsym+dt.Nundefsym] { + all = append(all, s.Name) + } + return all, nil +} + +// ImportedLibraries returns the paths of all libraries +// referred to by the binary f that are expected to be +// linked with the binary at dynamic link time. +func (f *File) ImportedLibraries() ([]string, os.Error) { + var all []string + for _, l := range f.Loads { + if lib, ok := l.(*Dylib); ok { + all = append(all, lib.Name) + } + } + return all, nil +} diff --git a/libgo/go/debug/macho/file_test.go b/libgo/go/debug/macho/file_test.go new file mode 100644 index 000000000..56d8a20be --- /dev/null +++ b/libgo/go/debug/macho/file_test.go @@ -0,0 +1,167 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package macho + +import ( + "reflect" + "testing" +) + +type fileTest struct { + file string + hdr FileHeader + segments []*SegmentHeader + sections []*SectionHeader +} + +var fileTests = []fileTest{ + { + "testdata/gcc-386-darwin-exec", + FileHeader{0xfeedface, Cpu386, 0x3, 0x2, 0xc, 0x3c0, 0x85}, + []*SegmentHeader{ + &SegmentHeader{LoadCmdSegment, 0x38, "__PAGEZERO", 0x0, 0x1000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + &SegmentHeader{LoadCmdSegment, 0xc0, "__TEXT", 0x1000, 0x1000, 0x0, 0x1000, 0x7, 0x5, 0x2, 0x0}, + &SegmentHeader{LoadCmdSegment, 0xc0, "__DATA", 0x2000, 0x1000, 0x1000, 0x1000, 0x7, 0x3, 0x2, 0x0}, + &SegmentHeader{LoadCmdSegment, 0x7c, "__IMPORT", 0x3000, 0x1000, 0x2000, 0x1000, 0x7, 0x7, 0x1, 0x0}, + &SegmentHeader{LoadCmdSegment, 0x38, "__LINKEDIT", 0x4000, 0x1000, 0x3000, 0x12c, 0x7, 0x1, 0x0, 0x0}, + nil, + nil, + nil, + nil, + nil, + nil, + nil, + }, + []*SectionHeader{ + &SectionHeader{"__text", "__TEXT", 0x1f68, 0x88, 0xf68, 0x2, 0x0, 0x0, 0x80000400}, + &SectionHeader{"__cstring", "__TEXT", 0x1ff0, 0xd, 0xff0, 0x0, 0x0, 0x0, 0x2}, + &SectionHeader{"__data", "__DATA", 0x2000, 0x14, 0x1000, 0x2, 0x0, 0x0, 0x0}, + &SectionHeader{"__dyld", "__DATA", 0x2014, 0x1c, 0x1014, 0x2, 0x0, 0x0, 0x0}, + &SectionHeader{"__jump_table", "__IMPORT", 0x3000, 0xa, 0x2000, 0x6, 0x0, 0x0, 0x4000008}, + }, + }, + { + "testdata/gcc-amd64-darwin-exec", + FileHeader{0xfeedfacf, CpuAmd64, 0x80000003, 0x2, 0xb, 0x568, 0x85}, + []*SegmentHeader{ + &SegmentHeader{LoadCmdSegment64, 0x48, "__PAGEZERO", 0x0, 0x100000000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + &SegmentHeader{LoadCmdSegment64, 0x1d8, "__TEXT", 0x100000000, 0x1000, 0x0, 0x1000, 0x7, 0x5, 0x5, 0x0}, + &SegmentHeader{LoadCmdSegment64, 0x138, "__DATA", 0x100001000, 0x1000, 0x1000, 0x1000, 0x7, 0x3, 0x3, 0x0}, + &SegmentHeader{LoadCmdSegment64, 0x48, "__LINKEDIT", 0x100002000, 0x1000, 0x2000, 0x140, 0x7, 0x1, 0x0, 0x0}, + nil, + nil, + nil, + nil, + nil, + nil, + nil, + }, + []*SectionHeader{ + &SectionHeader{"__text", "__TEXT", 0x100000f14, 0x6d, 0xf14, 0x2, 0x0, 0x0, 0x80000400}, + &SectionHeader{"__symbol_stub1", "__TEXT", 0x100000f81, 0xc, 0xf81, 0x0, 0x0, 0x0, 0x80000408}, + &SectionHeader{"__stub_helper", "__TEXT", 0x100000f90, 0x18, 0xf90, 0x2, 0x0, 0x0, 0x0}, + &SectionHeader{"__cstring", "__TEXT", 0x100000fa8, 0xd, 0xfa8, 0x0, 0x0, 0x0, 0x2}, + &SectionHeader{"__eh_frame", "__TEXT", 0x100000fb8, 0x48, 0xfb8, 0x3, 0x0, 0x0, 0x6000000b}, + &SectionHeader{"__data", "__DATA", 0x100001000, 0x1c, 0x1000, 0x3, 0x0, 0x0, 0x0}, + &SectionHeader{"__dyld", "__DATA", 0x100001020, 0x38, 0x1020, 0x3, 0x0, 0x0, 0x0}, + &SectionHeader{"__la_symbol_ptr", "__DATA", 0x100001058, 0x10, 0x1058, 0x2, 0x0, 0x0, 0x7}, + }, + }, + { + "testdata/gcc-amd64-darwin-exec-debug", + FileHeader{0xfeedfacf, CpuAmd64, 0x80000003, 0xa, 0x4, 0x5a0, 0}, + []*SegmentHeader{ + nil, + &SegmentHeader{LoadCmdSegment64, 0x1d8, "__TEXT", 0x100000000, 0x1000, 0x0, 0x0, 0x7, 0x5, 0x5, 0x0}, + &SegmentHeader{LoadCmdSegment64, 0x138, "__DATA", 0x100001000, 0x1000, 0x0, 0x0, 0x7, 0x3, 0x3, 0x0}, + &SegmentHeader{LoadCmdSegment64, 0x278, "__DWARF", 0x100002000, 0x1000, 0x1000, 0x1bc, 0x7, 0x3, 0x7, 0x0}, + }, + []*SectionHeader{ + &SectionHeader{"__text", "__TEXT", 0x100000f14, 0x0, 0x0, 0x2, 0x0, 0x0, 0x80000400}, + &SectionHeader{"__symbol_stub1", "__TEXT", 0x100000f81, 0x0, 0x0, 0x0, 0x0, 0x0, 0x80000408}, + &SectionHeader{"__stub_helper", "__TEXT", 0x100000f90, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0}, + &SectionHeader{"__cstring", "__TEXT", 0x100000fa8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2}, + &SectionHeader{"__eh_frame", "__TEXT", 0x100000fb8, 0x0, 0x0, 0x3, 0x0, 0x0, 0x6000000b}, + &SectionHeader{"__data", "__DATA", 0x100001000, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0}, + &SectionHeader{"__dyld", "__DATA", 0x100001020, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0}, + &SectionHeader{"__la_symbol_ptr", "__DATA", 0x100001058, 0x0, 0x0, 0x2, 0x0, 0x0, 0x7}, + &SectionHeader{"__debug_abbrev", "__DWARF", 0x100002000, 0x36, 0x1000, 0x0, 0x0, 0x0, 0x0}, + &SectionHeader{"__debug_aranges", "__DWARF", 0x100002036, 0x30, 0x1036, 0x0, 0x0, 0x0, 0x0}, + &SectionHeader{"__debug_frame", "__DWARF", 0x100002066, 0x40, 0x1066, 0x0, 0x0, 0x0, 0x0}, + &SectionHeader{"__debug_info", "__DWARF", 0x1000020a6, 0x54, 0x10a6, 0x0, 0x0, 0x0, 0x0}, + &SectionHeader{"__debug_line", "__DWARF", 0x1000020fa, 0x47, 0x10fa, 0x0, 0x0, 0x0, 0x0}, + &SectionHeader{"__debug_pubnames", "__DWARF", 0x100002141, 0x1b, 0x1141, 0x0, 0x0, 0x0, 0x0}, + &SectionHeader{"__debug_str", "__DWARF", 0x10000215c, 0x60, 0x115c, 0x0, 0x0, 0x0, 0x0}, + }, + }, +} + +func TestOpen(t *testing.T) { + for i := range fileTests { + tt := &fileTests[i] + + f, err := Open(tt.file) + if err != nil { + t.Error(err) + continue + } + if !reflect.DeepEqual(f.FileHeader, tt.hdr) { + t.Errorf("open %s:\n\thave %#v\n\twant %#v\n", tt.file, f.FileHeader, tt.hdr) + continue + } + for i, l := range f.Loads { + if i >= len(tt.segments) { + break + } + sh := tt.segments[i] + s, ok := l.(*Segment) + if sh == nil { + if ok { + t.Errorf("open %s, section %d: skipping %#v\n", tt.file, i, &s.SegmentHeader) + } + continue + } + if !ok { + t.Errorf("open %s, section %d: not *Segment\n", tt.file, i) + continue + } + have := &s.SegmentHeader + want := sh + if !reflect.DeepEqual(have, want) { + t.Errorf("open %s, segment %d:\n\thave %#v\n\twant %#v\n", tt.file, i, have, want) + } + } + tn := len(tt.segments) + fn := len(f.Loads) + if tn != fn { + t.Errorf("open %s: len(Loads) = %d, want %d", tt.file, fn, tn) + } + + for i, sh := range f.Sections { + if i >= len(tt.sections) { + break + } + have := &sh.SectionHeader + want := tt.sections[i] + if !reflect.DeepEqual(have, want) { + t.Errorf("open %s, section %d:\n\thave %#v\n\twant %#v\n", tt.file, i, have, want) + } + } + tn = len(tt.sections) + fn = len(f.Sections) + if tn != fn { + t.Errorf("open %s: len(Sections) = %d, want %d", tt.file, fn, tn) + } + + } +} + +func TestOpenFailure(t *testing.T) { + filename := "file.go" // not a Mach-O file + _, err := Open(filename) // don't crash + if err == nil { + t.Errorf("open %s: succeeded unexpectedly", filename) + } +} diff --git a/libgo/go/debug/macho/macho.go b/libgo/go/debug/macho/macho.go new file mode 100644 index 000000000..1386f5acf --- /dev/null +++ b/libgo/go/debug/macho/macho.go @@ -0,0 +1,305 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Mach-O header data structures +// http://developer.apple.com/mac/library/documentation/DeveloperTools/Conceptual/MachORuntime/Reference/reference.html + +package macho + +import "strconv" + +// A FileHeader represents a Mach-O file header. +type FileHeader struct { + Magic uint32 + Cpu Cpu + SubCpu uint32 + Type Type + Ncmd uint32 + Cmdsz uint32 + Flags uint32 +} + +const ( + fileHeaderSize32 = 7 * 4 + fileHeaderSize64 = 8 * 4 +) + +const ( + Magic32 uint32 = 0xfeedface + Magic64 uint32 = 0xfeedfacf +) + +// A Type is a Mach-O file type, either an object or an executable. +type Type uint32 + +const ( + TypeObj Type = 1 + TypeExec Type = 2 +) + +// A Cpu is a Mach-O cpu type. +type Cpu uint32 + +const ( + Cpu386 Cpu = 7 + CpuAmd64 Cpu = Cpu386 + 1<<24 +) + +var cpuStrings = []intName{ + {uint32(Cpu386), "Cpu386"}, + {uint32(CpuAmd64), "CpuAmd64"}, +} + +func (i Cpu) String() string { return stringName(uint32(i), cpuStrings, false) } +func (i Cpu) GoString() string { return stringName(uint32(i), cpuStrings, true) } + +// A LoadCmd is a Mach-O load command. +type LoadCmd uint32 + +const ( + LoadCmdSegment LoadCmd = 1 + LoadCmdSymtab LoadCmd = 2 + LoadCmdThread LoadCmd = 4 + LoadCmdUnixThread LoadCmd = 5 // thread+stack + LoadCmdDysymtab LoadCmd = 11 + LoadCmdDylib LoadCmd = 12 + LoadCmdDylinker LoadCmd = 15 + LoadCmdSegment64 LoadCmd = 25 +) + +var cmdStrings = []intName{ + {uint32(LoadCmdSegment), "LoadCmdSegment"}, + {uint32(LoadCmdThread), "LoadCmdThread"}, + {uint32(LoadCmdUnixThread), "LoadCmdUnixThread"}, + {uint32(LoadCmdDylib), "LoadCmdDylib"}, + {uint32(LoadCmdSegment64), "LoadCmdSegment64"}, +} + +func (i LoadCmd) String() string { return stringName(uint32(i), cmdStrings, false) } +func (i LoadCmd) GoString() string { return stringName(uint32(i), cmdStrings, true) } + +// A Segment64 is a 64-bit Mach-O segment load command. +type Segment64 struct { + Cmd LoadCmd + Len uint32 + Name [16]byte + Addr uint64 + Memsz uint64 + Offset uint64 + Filesz uint64 + Maxprot uint32 + Prot uint32 + Nsect uint32 + Flag uint32 +} + +// A Segment32 is a 32-bit Mach-O segment load command. +type Segment32 struct { + Cmd LoadCmd + Len uint32 + Name [16]byte + Addr uint32 + Memsz uint32 + Offset uint32 + Filesz uint32 + Maxprot uint32 + Prot uint32 + Nsect uint32 + Flag uint32 +} + +// A DylibCmd is a Mach-O load dynamic library command. +type DylibCmd struct { + Cmd LoadCmd + Len uint32 + Name uint32 + Time uint32 + CurrentVersion uint32 + CompatVersion uint32 +} + +// A Section32 is a 32-bit Mach-O section header. +type Section32 struct { + Name [16]byte + Seg [16]byte + Addr uint32 + Size uint32 + Offset uint32 + Align uint32 + Reloff uint32 + Nreloc uint32 + Flags uint32 + Reserve1 uint32 + Reserve2 uint32 +} + +// A Section32 is a 64-bit Mach-O section header. +type Section64 struct { + Name [16]byte + Seg [16]byte + Addr uint64 + Size uint64 + Offset uint32 + Align uint32 + Reloff uint32 + Nreloc uint32 + Flags uint32 + Reserve1 uint32 + Reserve2 uint32 + Reserve3 uint32 +} + +// A SymtabCmd is a Mach-O symbol table command. +type SymtabCmd struct { + Cmd LoadCmd + Len uint32 + Symoff uint32 + Nsyms uint32 + Stroff uint32 + Strsize uint32 +} + +// A DysymtabCmd is a Mach-O dynamic symbol table command. +type DysymtabCmd struct { + Cmd LoadCmd + Len uint32 + Ilocalsym uint32 + Nlocalsym uint32 + Iextdefsym uint32 + Nextdefsym uint32 + Iundefsym uint32 + Nundefsym uint32 + Tocoffset uint32 + Ntoc uint32 + Modtaboff uint32 + Nmodtab uint32 + Extrefsymoff uint32 + Nextrefsyms uint32 + Indirectsymoff uint32 + Nindirectsyms uint32 + Extreloff uint32 + Nextrel uint32 + Locreloff uint32 + Nlocrel uint32 +} + +// An Nlist32 is a Mach-O 32-bit symbol table entry. +type Nlist32 struct { + Name uint32 + Type uint8 + Sect uint8 + Desc uint16 + Value uint32 +} + +// An Nlist64 is a Mach-O 64-bit symbol table entry. +type Nlist64 struct { + Name uint32 + Type uint8 + Sect uint8 + Desc uint16 + Value uint64 +} + +// A Symbol is a Mach-O 32-bit or 64-bit symbol table entry. +type Symbol struct { + Name string + Type uint8 + Sect uint8 + Desc uint16 + Value uint64 +} + +// A Thread is a Mach-O thread state command. +type Thread struct { + Cmd LoadCmd + Len uint32 + Type uint32 + Data []uint32 +} + +// Regs386 is the Mach-O 386 register structure. +type Regs386 struct { + AX uint32 + BX uint32 + CX uint32 + DX uint32 + DI uint32 + SI uint32 + BP uint32 + SP uint32 + SS uint32 + FLAGS uint32 + IP uint32 + CS uint32 + DS uint32 + ES uint32 + FS uint32 + GS uint32 +} + +// RegsAMD64 is the Mach-O AMD64 register structure. +type RegsAMD64 struct { + AX uint64 + BX uint64 + CX uint64 + DX uint64 + DI uint64 + SI uint64 + BP uint64 + SP uint64 + R8 uint64 + R9 uint64 + R10 uint64 + R11 uint64 + R12 uint64 + R13 uint64 + R14 uint64 + R15 uint64 + IP uint64 + FLAGS uint64 + CS uint64 + FS uint64 + GS uint64 +} + +type intName struct { + i uint32 + s string +} + +func stringName(i uint32, names []intName, goSyntax bool) string { + for _, n := range names { + if n.i == i { + if goSyntax { + return "macho." + n.s + } + return n.s + } + } + return strconv.Uitoa64(uint64(i)) +} + +func flagName(i uint32, names []intName, goSyntax bool) string { + s := "" + for _, n := range names { + if n.i&i == n.i { + if len(s) > 0 { + s += "+" + } + if goSyntax { + s += "macho." + } + s += n.s + i -= n.i + } + } + if len(s) == 0 { + return "0x" + strconv.Uitob64(uint64(i), 16) + } + if i != 0 { + s += "+0x" + strconv.Uitob64(uint64(i), 16) + } + return s +} diff --git a/libgo/go/debug/macho/testdata/gcc-386-darwin-exec b/libgo/go/debug/macho/testdata/gcc-386-darwin-exec new file mode 100755 index 000000000..03ba1bafa Binary files /dev/null and b/libgo/go/debug/macho/testdata/gcc-386-darwin-exec differ diff --git a/libgo/go/debug/macho/testdata/gcc-amd64-darwin-exec b/libgo/go/debug/macho/testdata/gcc-amd64-darwin-exec new file mode 100755 index 000000000..5155a5a26 Binary files /dev/null and b/libgo/go/debug/macho/testdata/gcc-amd64-darwin-exec differ diff --git a/libgo/go/debug/macho/testdata/gcc-amd64-darwin-exec-debug b/libgo/go/debug/macho/testdata/gcc-amd64-darwin-exec-debug new file mode 100644 index 000000000..a47d3aef7 Binary files /dev/null and b/libgo/go/debug/macho/testdata/gcc-amd64-darwin-exec-debug differ diff --git a/libgo/go/debug/macho/testdata/hello.c b/libgo/go/debug/macho/testdata/hello.c new file mode 100644 index 000000000..a689d3644 --- /dev/null +++ b/libgo/go/debug/macho/testdata/hello.c @@ -0,0 +1,8 @@ +#include + +int +main(void) +{ + printf("hello, world\n"); + return 0; +} diff --git a/libgo/go/debug/pe/file.go b/libgo/go/debug/pe/file.go new file mode 100644 index 000000000..82c02407b --- /dev/null +++ b/libgo/go/debug/pe/file.go @@ -0,0 +1,309 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package pe implements access to PE (Microsoft Windows Portable Executable) files. +package pe + +import ( + "debug/dwarf" + "encoding/binary" + "fmt" + "io" + "os" + "strconv" +) + +// A File represents an open PE file. +type File struct { + FileHeader + Sections []*Section + + closer io.Closer +} + +type SectionHeader struct { + Name string + VirtualSize uint32 + VirtualAddress uint32 + Size uint32 + Offset uint32 + PointerToRelocations uint32 + PointerToLineNumbers uint32 + NumberOfRelocations uint16 + NumberOfLineNumbers uint16 + Characteristics uint32 +} + + +type Section struct { + SectionHeader + + // Embed ReaderAt for ReadAt method. + // Do not embed SectionReader directly + // to avoid having Read and Seek. + // If a client wants Read and Seek it must use + // Open() to avoid fighting over the seek offset + // with other clients. + io.ReaderAt + sr *io.SectionReader +} + +type ImportDirectory struct { + OriginalFirstThunk uint32 + TimeDateStamp uint32 + ForwarderChain uint32 + Name uint32 + FirstThunk uint32 + + dll string + rva []uint32 +} + +// Data reads and returns the contents of the PE section. +func (s *Section) Data() ([]byte, os.Error) { + dat := make([]byte, s.sr.Size()) + n, err := s.sr.ReadAt(dat, 0) + return dat[0:n], err +} + +// Open returns a new ReadSeeker reading the PE section. +func (s *Section) Open() io.ReadSeeker { return io.NewSectionReader(s.sr, 0, 1<<63-1) } + + +type FormatError struct { + off int64 + msg string + val interface{} +} + +func (e *FormatError) String() string { + msg := e.msg + if e.val != nil { + msg += fmt.Sprintf(" '%v'", e.val) + } + msg += fmt.Sprintf(" in record at byte %#x", e.off) + return msg +} + +// Open opens the named file using os.Open and prepares it for use as a PE binary. +func Open(name string) (*File, os.Error) { + f, err := os.Open(name, os.O_RDONLY, 0) + if err != nil { + return nil, err + } + ff, err := NewFile(f) + if err != nil { + f.Close() + return nil, err + } + ff.closer = f + return ff, nil +} + +// Close closes the File. +// If the File was created using NewFile directly instead of Open, +// Close has no effect. +func (f *File) Close() os.Error { + var err os.Error + if f.closer != nil { + err = f.closer.Close() + f.closer = nil + } + return err +} + +// NewFile creates a new File for acecssing a PE binary in an underlying reader. +func NewFile(r io.ReaderAt) (*File, os.Error) { + f := new(File) + sr := io.NewSectionReader(r, 0, 1<<63-1) + + var dosheader [96]byte + if _, err := r.ReadAt(dosheader[0:], 0); err != nil { + return nil, err + } + var base int64 + if dosheader[0] == 'M' && dosheader[1] == 'Z' { + var sign [4]byte + r.ReadAt(sign[0:], int64(dosheader[0x3c])) + if !(sign[0] == 'P' && sign[1] == 'E' && sign[2] == 0 && sign[3] == 0) { + return nil, os.NewError("Invalid PE File Format.") + } + base = int64(dosheader[0x3c]) + 4 + } else { + base = int64(0) + } + sr.Seek(base, 0) + if err := binary.Read(sr, binary.LittleEndian, &f.FileHeader); err != nil { + return nil, err + } + if f.FileHeader.Machine != IMAGE_FILE_MACHINE_UNKNOWN && f.FileHeader.Machine != IMAGE_FILE_MACHINE_AMD64 && f.FileHeader.Machine != IMAGE_FILE_MACHINE_I386 { + return nil, os.NewError("Invalid PE File Format.") + } + // get symbol string table + sr.Seek(int64(f.FileHeader.PointerToSymbolTable+18*f.FileHeader.NumberOfSymbols), 0) + var l uint32 + if err := binary.Read(sr, binary.LittleEndian, &l); err != nil { + return nil, err + } + ss := make([]byte, l) + if _, err := r.ReadAt(ss, int64(f.FileHeader.PointerToSymbolTable+18*f.FileHeader.NumberOfSymbols)); err != nil { + return nil, err + } + sr.Seek(base, 0) + binary.Read(sr, binary.LittleEndian, &f.FileHeader) + sr.Seek(int64(f.FileHeader.SizeOfOptionalHeader), 1) //Skip OptionalHeader + f.Sections = make([]*Section, f.FileHeader.NumberOfSections) + for i := 0; i < int(f.FileHeader.NumberOfSections); i++ { + sh := new(SectionHeader32) + if err := binary.Read(sr, binary.LittleEndian, sh); err != nil { + return nil, err + } + var name string + if sh.Name[0] == '\x2F' { + si, _ := strconv.Atoi(cstring(sh.Name[1:])) + name, _ = getString(ss, si) + } else { + name = cstring(sh.Name[0:]) + } + s := new(Section) + s.SectionHeader = SectionHeader{ + Name: name, + VirtualSize: uint32(sh.VirtualSize), + VirtualAddress: uint32(sh.VirtualAddress), + Size: uint32(sh.SizeOfRawData), + Offset: uint32(sh.PointerToRawData), + PointerToRelocations: uint32(sh.PointerToRelocations), + PointerToLineNumbers: uint32(sh.PointerToLineNumbers), + NumberOfRelocations: uint16(sh.NumberOfRelocations), + NumberOfLineNumbers: uint16(sh.NumberOfLineNumbers), + Characteristics: uint32(sh.Characteristics), + } + s.sr = io.NewSectionReader(r, int64(s.SectionHeader.Offset), int64(s.SectionHeader.Size)) + s.ReaderAt = s.sr + f.Sections[i] = s + } + return f, nil +} + +func cstring(b []byte) string { + var i int + for i = 0; i < len(b) && b[i] != 0; i++ { + } + return string(b[0:i]) +} + +// getString extracts a string from symbol string table. +func getString(section []byte, start int) (string, bool) { + if start < 0 || start >= len(section) { + return "", false + } + + for end := start; end < len(section); end++ { + if section[end] == 0 { + return string(section[start:end]), true + } + } + return "", false +} + +// Section returns the first section with the given name, or nil if no such +// section exists. +func (f *File) Section(name string) *Section { + for _, s := range f.Sections { + if s.Name == name { + return s + } + } + return nil +} + +func (f *File) DWARF() (*dwarf.Data, os.Error) { + // There are many other DWARF sections, but these + // are the required ones, and the debug/dwarf package + // does not use the others, so don't bother loading them. + var names = [...]string{"abbrev", "info", "str"} + var dat [len(names)][]byte + for i, name := range names { + name = ".debug_" + name + s := f.Section(name) + if s == nil { + continue + } + b, err := s.Data() + if err != nil && uint32(len(b)) < s.Size { + return nil, err + } + dat[i] = b + } + + abbrev, info, str := dat[0], dat[1], dat[2] + return dwarf.New(abbrev, nil, nil, info, nil, nil, nil, str) +} + +// ImportedSymbols returns the names of all symbols +// referred to by the binary f that are expected to be +// satisfied by other libraries at dynamic load time. +// It does not return weak symbols. +func (f *File) ImportedSymbols() ([]string, os.Error) { + ds := f.Section(".idata") + if ds == nil { + // not dynamic, so no libraries + return nil, nil + } + d, err := ds.Data() + if err != nil { + return nil, err + } + var ida []ImportDirectory + for len(d) > 0 { + var dt ImportDirectory + dt.OriginalFirstThunk = binary.LittleEndian.Uint32(d[0:4]) + dt.Name = binary.LittleEndian.Uint32(d[12:16]) + dt.FirstThunk = binary.LittleEndian.Uint32(d[16:20]) + d = d[20:] + if dt.OriginalFirstThunk == 0 { + break + } + ida = append(ida, dt) + } + for i, _ := range ida { + for len(d) > 0 { + va := binary.LittleEndian.Uint32(d[0:4]) + d = d[4:] + if va == 0 { + break + } + ida[i].rva = append(ida[i].rva, va) + } + } + for _, _ = range ida { + for len(d) > 0 { + va := binary.LittleEndian.Uint32(d[0:4]) + d = d[4:] + if va == 0 { + break + } + } + } + names, _ := ds.Data() + var all []string + for _, dt := range ida { + dt.dll, _ = getString(names, int(dt.Name-ds.VirtualAddress)) + for _, va := range dt.rva { + fn, _ := getString(names, int(va-ds.VirtualAddress+2)) + all = append(all, fn+":"+dt.dll) + } + } + + return all, nil +} + +// ImportedLibraries returns the names of all libraries +// referred to by the binary f that are expected to be +// linked with the binary at dynamic link time. +func (f *File) ImportedLibraries() ([]string, os.Error) { + // TODO + // cgo -dynimport don't use this for windows PE, so just return. + return nil, nil +} diff --git a/libgo/go/debug/pe/file_test.go b/libgo/go/debug/pe/file_test.go new file mode 100644 index 000000000..2c5c25b8c --- /dev/null +++ b/libgo/go/debug/pe/file_test.go @@ -0,0 +1,99 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package pe + +import ( + "reflect" + "testing" +) + +type fileTest struct { + file string + hdr FileHeader + sections []*SectionHeader +} + +var fileTests = []fileTest{ + { + "testdata/gcc-386-mingw-obj", + FileHeader{0x014c, 0x000c, 0x0, 0x64a, 0x1e, 0x0, 0x104}, + []*SectionHeader{ + &SectionHeader{".text", 0, 0, 36, 500, 1440, 0, 3, 0, 0x60300020}, + &SectionHeader{".data", 0, 0, 0, 0, 0, 0, 0, 0, 3224371264}, + &SectionHeader{".bss", 0, 0, 0, 0, 0, 0, 0, 0, 3224371328}, + &SectionHeader{".debug_abbrev", 0, 0, 137, 536, 0, 0, 0, 0, 0x42100000}, + &SectionHeader{".debug_info", 0, 0, 418, 673, 1470, 0, 7, 0, 1108344832}, + &SectionHeader{".debug_line", 0, 0, 128, 1091, 1540, 0, 1, 0, 1108344832}, + &SectionHeader{".rdata", 0, 0, 16, 1219, 0, 0, 0, 0, 1076887616}, + &SectionHeader{".debug_frame", 0, 0, 52, 1235, 1550, 0, 2, 0, 1110441984}, + &SectionHeader{".debug_loc", 0, 0, 56, 1287, 0, 0, 0, 0, 1108344832}, + &SectionHeader{".debug_pubnames", 0, 0, 27, 1343, 1570, 0, 1, 0, 1108344832}, + &SectionHeader{".debug_pubtypes", 0, 0, 38, 1370, 1580, 0, 1, 0, 1108344832}, + &SectionHeader{".debug_aranges", 0, 0, 32, 1408, 1590, 0, 2, 0, 1108344832}, + }, + }, + { + "testdata/gcc-386-mingw-exec", + FileHeader{0x014c, 0x000f, 0x4c6a1b60, 0x3c00, 0x282, 0xe0, 0x107}, + []*SectionHeader{ + &SectionHeader{Name: ".text", VirtualSize: 0xcd8, VirtualAddress: 0x1000, Size: 0xe00, Offset: 0x400, PointerToRelocations: 0x0, PointerToLineNumbers: 0x0, NumberOfRelocations: 0x0, NumberOfLineNumbers: 0x0, Characteristics: 0x60500060}, + &SectionHeader{Name: ".data", VirtualSize: 0x10, VirtualAddress: 0x2000, Size: 0x200, Offset: 0x1200, PointerToRelocations: 0x0, PointerToLineNumbers: 0x0, NumberOfRelocations: 0x0, NumberOfLineNumbers: 0x0, Characteristics: 0xc0300040}, + &SectionHeader{Name: ".rdata", VirtualSize: 0x120, VirtualAddress: 0x3000, Size: 0x200, Offset: 0x1400, PointerToRelocations: 0x0, PointerToLineNumbers: 0x0, NumberOfRelocations: 0x0, NumberOfLineNumbers: 0x0, Characteristics: 0x40300040}, + &SectionHeader{Name: ".bss", VirtualSize: 0xdc, VirtualAddress: 0x4000, Size: 0x0, Offset: 0x0, PointerToRelocations: 0x0, PointerToLineNumbers: 0x0, NumberOfRelocations: 0x0, NumberOfLineNumbers: 0x0, Characteristics: 0xc0400080}, + &SectionHeader{Name: ".idata", VirtualSize: 0x3c8, VirtualAddress: 0x5000, Size: 0x400, Offset: 0x1600, PointerToRelocations: 0x0, PointerToLineNumbers: 0x0, NumberOfRelocations: 0x0, NumberOfLineNumbers: 0x0, Characteristics: 0xc0300040}, + &SectionHeader{Name: ".CRT", VirtualSize: 0x18, VirtualAddress: 0x6000, Size: 0x200, Offset: 0x1a00, PointerToRelocations: 0x0, PointerToLineNumbers: 0x0, NumberOfRelocations: 0x0, NumberOfLineNumbers: 0x0, Characteristics: 0xc0300040}, + &SectionHeader{Name: ".tls", VirtualSize: 0x20, VirtualAddress: 0x7000, Size: 0x200, Offset: 0x1c00, PointerToRelocations: 0x0, PointerToLineNumbers: 0x0, NumberOfRelocations: 0x0, NumberOfLineNumbers: 0x0, Characteristics: 0xc0300040}, + &SectionHeader{Name: ".debug_aranges", VirtualSize: 0x20, VirtualAddress: 0x8000, Size: 0x200, Offset: 0x1e00, PointerToRelocations: 0x0, PointerToLineNumbers: 0x0, NumberOfRelocations: 0x0, NumberOfLineNumbers: 0x0, Characteristics: 0x42100000}, + &SectionHeader{Name: ".debug_pubnames", VirtualSize: 0x51, VirtualAddress: 0x9000, Size: 0x200, Offset: 0x2000, PointerToRelocations: 0x0, PointerToLineNumbers: 0x0, NumberOfRelocations: 0x0, NumberOfLineNumbers: 0x0, Characteristics: 0x42100000}, + &SectionHeader{Name: ".debug_pubtypes", VirtualSize: 0x91, VirtualAddress: 0xa000, Size: 0x200, Offset: 0x2200, PointerToRelocations: 0x0, PointerToLineNumbers: 0x0, NumberOfRelocations: 0x0, NumberOfLineNumbers: 0x0, Characteristics: 0x42100000}, + &SectionHeader{Name: ".debug_info", VirtualSize: 0xe22, VirtualAddress: 0xb000, Size: 0x1000, Offset: 0x2400, PointerToRelocations: 0x0, PointerToLineNumbers: 0x0, NumberOfRelocations: 0x0, NumberOfLineNumbers: 0x0, Characteristics: 0x42100000}, + &SectionHeader{Name: ".debug_abbrev", VirtualSize: 0x157, VirtualAddress: 0xc000, Size: 0x200, Offset: 0x3400, PointerToRelocations: 0x0, PointerToLineNumbers: 0x0, NumberOfRelocations: 0x0, NumberOfLineNumbers: 0x0, Characteristics: 0x42100000}, + &SectionHeader{Name: ".debug_line", VirtualSize: 0x144, VirtualAddress: 0xd000, Size: 0x200, Offset: 0x3600, PointerToRelocations: 0x0, PointerToLineNumbers: 0x0, NumberOfRelocations: 0x0, NumberOfLineNumbers: 0x0, Characteristics: 0x42100000}, + &SectionHeader{Name: ".debug_frame", VirtualSize: 0x34, VirtualAddress: 0xe000, Size: 0x200, Offset: 0x3800, PointerToRelocations: 0x0, PointerToLineNumbers: 0x0, NumberOfRelocations: 0x0, NumberOfLineNumbers: 0x0, Characteristics: 0x42300000}, + &SectionHeader{Name: ".debug_loc", VirtualSize: 0x38, VirtualAddress: 0xf000, Size: 0x200, Offset: 0x3a00, PointerToRelocations: 0x0, PointerToLineNumbers: 0x0, NumberOfRelocations: 0x0, NumberOfLineNumbers: 0x0, Characteristics: 0x42100000}, + }, + }, +} + +func TestOpen(t *testing.T) { + for i := range fileTests { + tt := &fileTests[i] + + f, err := Open(tt.file) + if err != nil { + t.Error(err) + continue + } + if !reflect.DeepEqual(f.FileHeader, tt.hdr) { + t.Errorf("open %s:\n\thave %#v\n\twant %#v\n", tt.file, f.FileHeader, tt.hdr) + continue + } + + for i, sh := range f.Sections { + if i >= len(tt.sections) { + break + } + have := &sh.SectionHeader + want := tt.sections[i] + if !reflect.DeepEqual(have, want) { + t.Errorf("open %s, section %d:\n\thave %#v\n\twant %#v\n", tt.file, i, have, want) + } + } + tn := len(tt.sections) + fn := len(f.Sections) + if tn != fn { + t.Errorf("open %s: len(Sections) = %d, want %d", tt.file, fn, tn) + } + + } +} + +func TestOpenFailure(t *testing.T) { + filename := "file.go" // not a PE file + _, err := Open(filename) // don't crash + if err == nil { + t.Errorf("open %s: succeeded unexpectedly", filename) + } +} diff --git a/libgo/go/debug/pe/pe.go b/libgo/go/debug/pe/pe.go new file mode 100644 index 000000000..b3dab739a --- /dev/null +++ b/libgo/go/debug/pe/pe.go @@ -0,0 +1,51 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package pe + +type FileHeader struct { + Machine uint16 + NumberOfSections uint16 + TimeDateStamp uint32 + PointerToSymbolTable uint32 + NumberOfSymbols uint32 + SizeOfOptionalHeader uint16 + Characteristics uint16 +} + +type SectionHeader32 struct { + Name [8]uint8 + VirtualSize uint32 + VirtualAddress uint32 + SizeOfRawData uint32 + PointerToRawData uint32 + PointerToRelocations uint32 + PointerToLineNumbers uint32 + NumberOfRelocations uint16 + NumberOfLineNumbers uint16 + Characteristics uint32 +} + +const ( + IMAGE_FILE_MACHINE_UNKNOWN = 0x0 + IMAGE_FILE_MACHINE_AM33 = 0x1d3 + IMAGE_FILE_MACHINE_AMD64 = 0x8664 + IMAGE_FILE_MACHINE_ARM = 0x1c0 + IMAGE_FILE_MACHINE_EBC = 0xebc + IMAGE_FILE_MACHINE_I386 = 0x14c + IMAGE_FILE_MACHINE_IA64 = 0x200 + IMAGE_FILE_MACHINE_M32R = 0x9041 + IMAGE_FILE_MACHINE_MIPS16 = 0x266 + IMAGE_FILE_MACHINE_MIPSFPU = 0x366 + IMAGE_FILE_MACHINE_MIPSFPU16 = 0x466 + IMAGE_FILE_MACHINE_POWERPC = 0x1f0 + IMAGE_FILE_MACHINE_POWERPCFP = 0x1f1 + IMAGE_FILE_MACHINE_R4000 = 0x166 + IMAGE_FILE_MACHINE_SH3 = 0x1a2 + IMAGE_FILE_MACHINE_SH3DSP = 0x1a3 + IMAGE_FILE_MACHINE_SH4 = 0x1a6 + IMAGE_FILE_MACHINE_SH5 = 0x1a8 + IMAGE_FILE_MACHINE_THUMB = 0x1c2 + IMAGE_FILE_MACHINE_WCEMIPSV2 = 0x169 +) diff --git a/libgo/go/debug/pe/testdata/gcc-386-mingw-exec b/libgo/go/debug/pe/testdata/gcc-386-mingw-exec new file mode 100644 index 000000000..4b808d043 Binary files /dev/null and b/libgo/go/debug/pe/testdata/gcc-386-mingw-exec differ diff --git a/libgo/go/debug/pe/testdata/gcc-386-mingw-obj b/libgo/go/debug/pe/testdata/gcc-386-mingw-obj new file mode 100644 index 000000000..0c84d898d Binary files /dev/null and b/libgo/go/debug/pe/testdata/gcc-386-mingw-obj differ diff --git a/libgo/go/debug/pe/testdata/hello.c b/libgo/go/debug/pe/testdata/hello.c new file mode 100644 index 000000000..a689d3644 --- /dev/null +++ b/libgo/go/debug/pe/testdata/hello.c @@ -0,0 +1,8 @@ +#include + +int +main(void) +{ + printf("hello, world\n"); + return 0; +} diff --git a/libgo/go/debug/proc/proc.go b/libgo/go/debug/proc/proc.go new file mode 100644 index 000000000..d89649cf8 --- /dev/null +++ b/libgo/go/debug/proc/proc.go @@ -0,0 +1,222 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package proc provides a platform-independent interface for +// tracing and controlling running processes. It supports +// multi-threaded processes and provides typical low-level debugging +// controls such as breakpoints, single stepping, and manipulating +// memory and registers. +package proc + +// TODO(rsc): Have to import everything that proc_linux.go +// and proc_darwin.go do, because deps.bash only looks at +// this file. +import ( + _ "container/vector" + _ "fmt" + _ "io" + "os" + _ "runtime" + "strconv" + _ "strings" + _ "sync" + _ "syscall" +) + +type Word uint64 + +// A Cause explains why a thread is stopped. +type Cause interface { + String() string +} + +// Regs is a set of named machine registers, including a program +// counter, link register, and stack pointer. +// +// TODO(austin) There's quite a proliferation of methods here. We +// could make a Reg interface with Get and Set and make this just PC, +// Link, SP, Names, and Reg. We could also put Index in Reg and that +// makes it easy to get the index of things like the PC (currently +// there's just no way to know that). This would also let us include +// other per-register information like how to print it. +type Regs interface { + // PC returns the value of the program counter. + PC() Word + + // SetPC sets the program counter to val. + SetPC(val Word) os.Error + + // Link returns the link register, if any. + Link() Word + + // SetLink sets the link register to val. + SetLink(val Word) os.Error + + // SP returns the value of the stack pointer. + SP() Word + + // SetSP sets the stack pointer register to val. + SetSP(val Word) os.Error + + // Names returns the names of all of the registers. + Names() []string + + // Get returns the value of a register, where i corresponds to + // the index of the register's name in the array returned by + // Names. + Get(i int) Word + + // Set sets the value of a register. + Set(i int, val Word) os.Error +} + +// Thread is a thread in the process being traced. +type Thread interface { + // Step steps this thread by a single instruction. The thread + // must be stopped. If the thread is currently stopped on a + // breakpoint, this will step over the breakpoint. + // + // XXX What if it's stopped because of a signal? + Step() os.Error + + // Stopped returns the reason that this thread is stopped. It + // is an error is the thread not stopped. + Stopped() (Cause, os.Error) + + // Regs retrieves the current register values from this + // thread. The thread must be stopped. + Regs() (Regs, os.Error) + + // Peek reads len(out) bytes from the address addr in this + // thread into out. The thread must be stopped. It returns + // the number of bytes successfully read. If an error occurs, + // such as attempting to read unmapped memory, this count + // could be short and an error will be returned. If this does + // encounter unmapped memory, it will read up to the byte + // preceding the unmapped area. + Peek(addr Word, out []byte) (int, os.Error) + + // Poke writes b to the address addr in this thread. The + // thread must be stopped. It returns the number of bytes + // successfully written. If an error occurs, such as + // attempting to write to unmapped memory, this count could be + // short and an error will be returned. If this does + // encounter unmapped memory, it will write up to the byte + // preceding the unmapped area. + Poke(addr Word, b []byte) (int, os.Error) +} + +// Process is a process being traced. It consists of a set of +// threads. A process can be running, stopped, or terminated. The +// process's state extends to all of its threads. +type Process interface { + // Threads returns an array of all threads in this process. + Threads() []Thread + + // AddBreakpoint creates a new breakpoint at program counter + // pc. Breakpoints can only be created when the process is + // stopped. It is an error if a breakpoint already exists at + // pc. + AddBreakpoint(pc Word) os.Error + + // RemoveBreakpoint removes the breakpoint at the program + // counter pc. It is an error if no breakpoint exists at pc. + RemoveBreakpoint(pc Word) os.Error + + // Stop stops all running threads in this process before + // returning. + Stop() os.Error + + // Continue resumes execution of all threads in this process. + // Any thread that is stopped on a breakpoint will be stepped + // over that breakpoint. Any thread that is stopped because + // of a signal (other than SIGSTOP or SIGTRAP) will receive + // the pending signal. + Continue() os.Error + + // WaitStop waits until all threads in process p are stopped + // as a result of some thread hitting a breakpoint, receiving + // a signal, creating a new thread, or exiting. + WaitStop() os.Error + + // Detach detaches from this process. All stopped threads + // will be resumed. + Detach() os.Error +} + +// Stopped is a stop cause used for threads that are stopped either by +// user request (e.g., from the Stop method or after single stepping), +// or that are stopped because some other thread caused the program to +// stop. +type Stopped struct{} + +func (c Stopped) String() string { return "stopped" } + +// Breakpoint is a stop cause resulting from a thread reaching a set +// breakpoint. +type Breakpoint Word + +// PC returns the program counter that the program is stopped at. +func (c Breakpoint) PC() Word { return Word(c) } + +func (c Breakpoint) String() string { + return "breakpoint at 0x" + strconv.Uitob64(uint64(c.PC()), 16) +} + +// Signal is a stop cause resulting from a thread receiving a signal. +// When the process is continued, the signal will be delivered. +type Signal string + +// Signal returns the signal being delivered to the thread. +func (c Signal) Name() string { return string(c) } + +func (c Signal) String() string { return c.Name() } + +// ThreadCreate is a stop cause returned from an existing thread when +// it creates a new thread. The new thread exists in a primordial +// form at this point and will begin executing in earnest when the +// process is continued. +type ThreadCreate struct { + thread Thread +} + +func (c *ThreadCreate) NewThread() Thread { return c.thread } + +func (c *ThreadCreate) String() string { return "thread create" } + +// ThreadExit is a stop cause resulting from a thread exiting. When +// this cause first arises, the thread will still be in the list of +// process threads and its registers and memory will still be +// accessible. +type ThreadExit struct { + exitStatus int + signal string +} + +// Exited returns true if the thread exited normally. +func (c *ThreadExit) Exited() bool { return c.exitStatus != -1 } + +// ExitStatus returns the exit status of the thread if it exited +// normally or -1 otherwise. +func (c *ThreadExit) ExitStatus() int { return c.exitStatus } + +// Signaled returns true if the thread was terminated by a signal. +func (c *ThreadExit) Signaled() bool { return c.exitStatus == -1 } + +// StopSignal returns the signal that terminated the thread, or "" if +// it was not terminated by a signal. +func (c *ThreadExit) StopSignal() string { return c.signal } + +func (c *ThreadExit) String() string { + res := "thread exited " + switch { + case c.Exited(): + res += "with status " + strconv.Itoa(c.ExitStatus()) + case c.Signaled(): + res += "from signal " + c.StopSignal() + default: + res += "from unknown cause" + } + return res +} diff --git a/libgo/go/debug/proc/proc_darwin.go b/libgo/go/debug/proc/proc_darwin.go new file mode 100644 index 000000000..7caf3a21a --- /dev/null +++ b/libgo/go/debug/proc/proc_darwin.go @@ -0,0 +1,17 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package proc + +import "os" + +// Process tracing is not supported on OS X yet. + +func Attach(pid int) (Process, os.Error) { + return nil, os.NewError("debug/proc not implemented on OS X") +} + +func ForkExec(argv0 string, argv []string, envv []string, dir string, fd []*os.File) (Process, os.Error) { + return Attach(0) +} diff --git a/libgo/go/debug/proc/proc_freebsd.go b/libgo/go/debug/proc/proc_freebsd.go new file mode 100644 index 000000000..f6474ce80 --- /dev/null +++ b/libgo/go/debug/proc/proc_freebsd.go @@ -0,0 +1,17 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package proc + +import "os" + +// Process tracing is not supported on FreeBSD yet. + +func Attach(pid int) (Process, os.Error) { + return nil, os.NewError("debug/proc not implemented on FreeBSD") +} + +func ForkExec(argv0 string, argv []string, envv []string, dir string, fd []*os.File) (Process, os.Error) { + return Attach(0) +} diff --git a/libgo/go/debug/proc/proc_linux.go b/libgo/go/debug/proc/proc_linux.go new file mode 100644 index 000000000..f0cc43a10 --- /dev/null +++ b/libgo/go/debug/proc/proc_linux.go @@ -0,0 +1,1316 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package proc + +// TODO(rsc): Imports here after to be in proc.go too in order +// for deps.bash to get the right answer. +import ( + "container/vector" + "fmt" + "io/ioutil" + "os" + "runtime" + "strconv" + "strings" + "sync" + "syscall" +) + +// This is an implementation of the process tracing interface using +// Linux's ptrace(2) interface. The implementation is multi-threaded. +// Each attached process has an associated monitor thread, and each +// running attached thread has an associated "wait" thread. The wait +// thread calls wait4 on the thread's TID and reports any wait events +// or errors via "debug events". The monitor thread consumes these +// wait events and updates the internally maintained state of each +// thread. All ptrace calls must run in the monitor thread, so the +// monitor executes closures received on the debugReq channel. +// +// As ptrace's documentation is somewhat light, this is heavily based +// on information gleaned from the implementation of ptrace found at +// http://lxr.linux.no/linux+v2.6.30/kernel/ptrace.c +// http://lxr.linux.no/linux+v2.6.30/arch/x86/kernel/ptrace.c#L854 +// as well as experimentation and examination of gdb's behavior. + +const ( + trace = false + traceIP = false + traceMem = false +) + +/* + * Thread state + */ + +// Each thread can be in one of the following set of states. +// Each state satisfies +// isRunning() || isStopped() || isZombie() || isTerminal(). +// +// Running threads can be sent signals and must be waited on, but they +// cannot be inspected using ptrace. +// +// Stopped threads can be inspected and continued, but cannot be +// meaningfully waited on. They can be sent signals, but the signals +// will be queued until they are running again. +// +// Zombie threads cannot be inspected, continued, or sent signals (and +// therefore they cannot be stopped), but they must be waited on. +// +// Terminal threads no longer exist in the OS and thus you can't do +// anything with them. +type threadState string + +const ( + running threadState = "Running" + singleStepping threadState = "SingleStepping" // Transient + stopping threadState = "Stopping" // Transient + stopped threadState = "Stopped" + stoppedBreakpoint threadState = "StoppedBreakpoint" + stoppedSignal threadState = "StoppedSignal" + stoppedThreadCreate threadState = "StoppedThreadCreate" + stoppedExiting threadState = "StoppedExiting" + exiting threadState = "Exiting" // Transient (except main thread) + exited threadState = "Exited" + detached threadState = "Detached" +) + +func (ts threadState) isRunning() bool { + return ts == running || ts == singleStepping || ts == stopping +} + +func (ts threadState) isStopped() bool { + return ts == stopped || ts == stoppedBreakpoint || ts == stoppedSignal || ts == stoppedThreadCreate || ts == stoppedExiting +} + +func (ts threadState) isZombie() bool { return ts == exiting } + +func (ts threadState) isTerminal() bool { return ts == exited || ts == detached } + +func (ts threadState) String() string { return string(ts) } + +/* + * Basic types + */ + +// A breakpoint stores information about a single breakpoint, +// including its program counter, the overwritten text if the +// breakpoint is installed. +type breakpoint struct { + pc uintptr + olddata []byte +} + +func (bp *breakpoint) String() string { + if bp == nil { + return "" + } + return fmt.Sprintf("%#x", bp.pc) +} + +// bpinst386 is the breakpoint instruction used on 386 and amd64. +var bpinst386 = []byte{0xcc} + +// A debugEvent represents a reason a thread stopped or a wait error. +type debugEvent struct { + *os.Waitmsg + t *thread + err os.Error +} + +// A debugReq is a request to execute a closure in the monitor thread. +type debugReq struct { + f func() os.Error + res chan os.Error +} + +// A transitionHandler specifies a function to be called when a thread +// changes state and a function to be called when an error occurs in +// the monitor. Both run in the monitor thread. Before the monitor +// invokes a handler, it removes the handler from the handler queue. +// The handler should re-add itself if needed. +type transitionHandler struct { + handle func(*thread, threadState, threadState) + onErr func(os.Error) +} + +// A process is a Linux process, which consists of a set of threads. +// Each running process has one monitor thread, which processes +// messages from the debugEvents, debugReqs, and stopReq channels and +// calls transition handlers. +// +// To send a message to the monitor thread, first receive from the +// ready channel. If the ready channel returns true, the monitor is +// still running and will accept a message. If the ready channel +// returns false, the monitor is not running (the ready channel has +// been closed), and the reason it is not running will be stored in err. +type process struct { + pid int + threads map[int]*thread + breakpoints map[uintptr]*breakpoint + ready chan bool + debugEvents chan *debugEvent + debugReqs chan *debugReq + stopReq chan os.Error + transitionHandlers vector.Vector + err os.Error +} + +// A thread represents a Linux thread in another process that is being +// debugged. Each running thread has an associated goroutine that +// waits for thread updates and sends them to the process monitor. +type thread struct { + tid int + proc *process + // Whether to ignore the next SIGSTOP received by wait. + ignoreNextSigstop bool + + // Thread state. Only modified via setState. + state threadState + // If state == StoppedBreakpoint + breakpoint *breakpoint + // If state == StoppedSignal or state == Exited + signal int + // If state == StoppedThreadCreate + newThread *thread + // If state == Exited + exitStatus int +} + +/* + * Errors + */ + +type badState struct { + thread *thread + message string + state threadState +} + +func (e *badState) String() string { + return fmt.Sprintf("Thread %d %s from state %v", e.thread.tid, e.message, e.state) +} + +type breakpointExistsError Word + +func (e breakpointExistsError) String() string { + return fmt.Sprintf("breakpoint already exists at PC %#x", e) +} + +type noBreakpointError Word + +func (e noBreakpointError) String() string { return fmt.Sprintf("no breakpoint at PC %#x", e) } + +type newThreadError struct { + *os.Waitmsg + wantPid int + wantSig int +} + +func (e *newThreadError) String() string { + return fmt.Sprintf("newThread wait wanted pid %v and signal %v, got %v and %v", e.Pid, e.StopSignal(), e.wantPid, e.wantSig) +} + +type ProcessExited struct{} + +func (p ProcessExited) String() string { return "process exited" } + +/* + * Ptrace wrappers + */ + +func (t *thread) ptracePeekText(addr uintptr, out []byte) (int, os.Error) { + c, err := syscall.PtracePeekText(t.tid, addr, out) + if traceMem { + fmt.Printf("peek(%#x) => %v, %v\n", addr, out, err) + } + return c, os.NewSyscallError("ptrace(PEEKTEXT)", err) +} + +func (t *thread) ptracePokeText(addr uintptr, out []byte) (int, os.Error) { + c, err := syscall.PtracePokeText(t.tid, addr, out) + if traceMem { + fmt.Printf("poke(%#x, %v) => %v\n", addr, out, err) + } + return c, os.NewSyscallError("ptrace(POKETEXT)", err) +} + +func (t *thread) ptraceGetRegs(regs *syscall.PtraceRegs) os.Error { + err := syscall.PtraceGetRegs(t.tid, regs) + return os.NewSyscallError("ptrace(GETREGS)", err) +} + +func (t *thread) ptraceSetRegs(regs *syscall.PtraceRegs) os.Error { + err := syscall.PtraceSetRegs(t.tid, regs) + return os.NewSyscallError("ptrace(SETREGS)", err) +} + +func (t *thread) ptraceSetOptions(options int) os.Error { + err := syscall.PtraceSetOptions(t.tid, options) + return os.NewSyscallError("ptrace(SETOPTIONS)", err) +} + +func (t *thread) ptraceGetEventMsg() (uint, os.Error) { + msg, err := syscall.PtraceGetEventMsg(t.tid) + return msg, os.NewSyscallError("ptrace(GETEVENTMSG)", err) +} + +func (t *thread) ptraceCont() os.Error { + err := syscall.PtraceCont(t.tid, 0) + return os.NewSyscallError("ptrace(CONT)", err) +} + +func (t *thread) ptraceContWithSignal(sig int) os.Error { + err := syscall.PtraceCont(t.tid, sig) + return os.NewSyscallError("ptrace(CONT)", err) +} + +func (t *thread) ptraceStep() os.Error { + err := syscall.PtraceSingleStep(t.tid) + return os.NewSyscallError("ptrace(SINGLESTEP)", err) +} + +func (t *thread) ptraceDetach() os.Error { + err := syscall.PtraceDetach(t.tid) + return os.NewSyscallError("ptrace(DETACH)", err) +} + +/* + * Logging utilties + */ + +var logLock sync.Mutex + +func (t *thread) logTrace(format string, args ...interface{}) { + if !trace { + return + } + logLock.Lock() + defer logLock.Unlock() + fmt.Fprintf(os.Stderr, "Thread %d", t.tid) + if traceIP { + var regs syscall.PtraceRegs + err := t.ptraceGetRegs(®s) + if err == nil { + fmt.Fprintf(os.Stderr, "@%x", regs.PC()) + } + } + fmt.Fprint(os.Stderr, ": ") + fmt.Fprintf(os.Stderr, format, args...) + fmt.Fprint(os.Stderr, "\n") +} + +func (t *thread) warn(format string, args ...interface{}) { + logLock.Lock() + defer logLock.Unlock() + fmt.Fprintf(os.Stderr, "Thread %d: WARNING ", t.tid) + fmt.Fprintf(os.Stderr, format, args...) + fmt.Fprint(os.Stderr, "\n") +} + +func (p *process) logTrace(format string, args ...interface{}) { + if !trace { + return + } + logLock.Lock() + defer logLock.Unlock() + fmt.Fprintf(os.Stderr, "Process %d: ", p.pid) + fmt.Fprintf(os.Stderr, format, args...) + fmt.Fprint(os.Stderr, "\n") +} + +/* + * State utilities + */ + +// someStoppedThread returns a stopped thread from the process. +// Returns nil if no threads are stopped. +// +// Must be called from the monitor thread. +func (p *process) someStoppedThread() *thread { + for _, t := range p.threads { + if t.state.isStopped() { + return t + } + } + return nil +} + +// someRunningThread returns a running thread from the process. +// Returns nil if no threads are running. +// +// Must be called from the monitor thread. +func (p *process) someRunningThread() *thread { + for _, t := range p.threads { + if t.state.isRunning() { + return t + } + } + return nil +} + +/* + * Breakpoint utilities + */ + +// installBreakpoints adds breakpoints to the attached process. +// +// Must be called from the monitor thread. +func (p *process) installBreakpoints() os.Error { + n := 0 + main := p.someStoppedThread() + for _, b := range p.breakpoints { + if b.olddata != nil { + continue + } + + b.olddata = make([]byte, len(bpinst386)) + _, err := main.ptracePeekText(uintptr(b.pc), b.olddata) + if err != nil { + b.olddata = nil + return err + } + + _, err = main.ptracePokeText(uintptr(b.pc), bpinst386) + if err != nil { + b.olddata = nil + return err + } + n++ + } + if n > 0 { + p.logTrace("installed %d/%d breakpoints", n, len(p.breakpoints)) + } + + return nil +} + +// uninstallBreakpoints removes the installed breakpoints from p. +// +// Must be called from the monitor thread. +func (p *process) uninstallBreakpoints() os.Error { + if len(p.threads) == 0 { + return nil + } + n := 0 + main := p.someStoppedThread() + for _, b := range p.breakpoints { + if b.olddata == nil { + continue + } + + _, err := main.ptracePokeText(uintptr(b.pc), b.olddata) + if err != nil { + return err + } + b.olddata = nil + n++ + } + if n > 0 { + p.logTrace("uninstalled %d/%d breakpoints", n, len(p.breakpoints)) + } + + return nil +} + +/* + * Debug event handling + */ + +// wait waits for a wait event from this thread and sends it on the +// debug events channel for this thread's process. This should be +// started in its own goroutine when the attached thread enters a +// running state. The goroutine will exit as soon as it sends a debug +// event. +func (t *thread) wait() { + for { + var ev debugEvent + ev.t = t + t.logTrace("beginning wait") + ev.Waitmsg, ev.err = os.Wait(t.tid, syscall.WALL) + if ev.err == nil && ev.Pid != t.tid { + panic(fmt.Sprint("Wait returned pid ", ev.Pid, " wanted ", t.tid)) + } + if ev.StopSignal() == syscall.SIGSTOP && t.ignoreNextSigstop { + // Spurious SIGSTOP. See Thread.Stop(). + t.ignoreNextSigstop = false + err := t.ptraceCont() + if err == nil { + continue + } + // If we failed to continue, just let + // the stop go through so we can + // update the thread's state. + } + if !<-t.proc.ready { + // The monitor exited + break + } + t.proc.debugEvents <- &ev + break + } +} + +// setState sets this thread's state, starts a wait thread if +// necessary, and invokes state transition handlers. +// +// Must be called from the monitor thread. +func (t *thread) setState(newState threadState) { + oldState := t.state + t.state = newState + t.logTrace("state %v -> %v", oldState, newState) + + if !oldState.isRunning() && (newState.isRunning() || newState.isZombie()) { + // Start waiting on this thread + go t.wait() + } + + // Invoke state change handlers + handlers := t.proc.transitionHandlers + if handlers.Len() == 0 { + return + } + + t.proc.transitionHandlers = nil + for _, h := range handlers { + h := h.(*transitionHandler) + h.handle(t, oldState, newState) + } +} + +// sendSigstop sends a SIGSTOP to this thread. +func (t *thread) sendSigstop() os.Error { + t.logTrace("sending SIGSTOP") + err := syscall.Tgkill(t.proc.pid, t.tid, syscall.SIGSTOP) + return os.NewSyscallError("tgkill", err) +} + +// stopAsync sends SIGSTOP to all threads in state 'running'. +// +// Must be called from the monitor thread. +func (p *process) stopAsync() os.Error { + for _, t := range p.threads { + if t.state == running { + err := t.sendSigstop() + if err != nil { + return err + } + t.setState(stopping) + } + } + return nil +} + +// doTrap handles SIGTRAP debug events with a cause of 0. These can +// be caused either by an installed breakpoint, a breakpoint in the +// program text, or by single stepping. +// +// TODO(austin) I think we also get this on an execve syscall. +func (ev *debugEvent) doTrap() (threadState, os.Error) { + t := ev.t + + if t.state == singleStepping { + return stopped, nil + } + + // Hit a breakpoint. Linux leaves the program counter after + // the breakpoint. If this is an installed breakpoint, we + // need to back the PC up to the breakpoint PC. + var regs syscall.PtraceRegs + err := t.ptraceGetRegs(®s) + if err != nil { + return stopped, err + } + + b, ok := t.proc.breakpoints[uintptr(regs.PC())-uintptr(len(bpinst386))] + if !ok { + // We must have hit a breakpoint that was actually in + // the program. Leave the IP where it is so we don't + // re-execute the breakpoint instruction. Expose the + // fact that we stopped with a SIGTRAP. + return stoppedSignal, nil + } + + t.breakpoint = b + t.logTrace("at breakpoint %v, backing up PC from %#x", b, regs.PC()) + + regs.SetPC(uint64(b.pc)) + err = t.ptraceSetRegs(®s) + if err != nil { + return stopped, err + } + return stoppedBreakpoint, nil +} + +// doPtraceClone handles SIGTRAP debug events with a PTRACE_EVENT_CLONE +// cause. It initializes the new thread, adds it to the process, and +// returns the appropriate thread state for the existing thread. +func (ev *debugEvent) doPtraceClone() (threadState, os.Error) { + t := ev.t + + // Get the TID of the new thread + tid, err := t.ptraceGetEventMsg() + if err != nil { + return stopped, err + } + + nt, err := t.proc.newThread(int(tid), syscall.SIGSTOP, true) + if err != nil { + return stopped, err + } + + // Remember the thread + t.newThread = nt + + return stoppedThreadCreate, nil +} + +// doPtraceExit handles SIGTRAP debug events with a PTRACE_EVENT_EXIT +// cause. It sets up the thread's state, but does not remove it from +// the process. A later WIFEXITED debug event will remove it from the +// process. +func (ev *debugEvent) doPtraceExit() (threadState, os.Error) { + t := ev.t + + // Get exit status + exitStatus, err := t.ptraceGetEventMsg() + if err != nil { + return stopped, err + } + ws := syscall.WaitStatus(exitStatus) + t.logTrace("exited with %v", ws) + switch { + case ws.Exited(): + t.exitStatus = ws.ExitStatus() + case ws.Signaled(): + t.signal = ws.Signal() + } + + // We still need to continue this thread and wait on this + // thread's WIFEXITED event. We'll delete it then. + return stoppedExiting, nil +} + +// process handles a debug event. It modifies any thread or process +// state as necessary, uninstalls breakpoints if necessary, and stops +// any running threads. +func (ev *debugEvent) process() os.Error { + if ev.err != nil { + return ev.err + } + + t := ev.t + t.exitStatus = -1 + t.signal = -1 + + // Decode wait status. + var state threadState + switch { + case ev.Stopped(): + state = stoppedSignal + t.signal = ev.StopSignal() + t.logTrace("stopped with %v", ev) + if ev.StopSignal() == syscall.SIGTRAP { + // What caused the debug trap? + var err os.Error + switch cause := ev.TrapCause(); cause { + case 0: + // Breakpoint or single stepping + state, err = ev.doTrap() + + case syscall.PTRACE_EVENT_CLONE: + state, err = ev.doPtraceClone() + + case syscall.PTRACE_EVENT_EXIT: + state, err = ev.doPtraceExit() + + default: + t.warn("Unknown trap cause %d", cause) + } + + if err != nil { + t.setState(stopped) + t.warn("failed to handle trap %v: %v", ev, err) + } + } + + case ev.Exited(): + state = exited + t.proc.threads[t.tid] = nil, false + t.logTrace("exited %v", ev) + // We should have gotten the exit status in + // PTRACE_EVENT_EXIT, but just in case. + t.exitStatus = ev.ExitStatus() + + case ev.Signaled(): + state = exited + t.proc.threads[t.tid] = nil, false + t.logTrace("signaled %v", ev) + // Again, this should be redundant. + t.signal = ev.Signal() + + default: + panic(fmt.Sprintf("Unexpected wait status %v", ev.Waitmsg)) + } + + // If we sent a SIGSTOP to the thread (indicated by state + // Stopping), we might have raced with a different type of + // stop. If we didn't get the stop we expected, then the + // SIGSTOP we sent is now queued up, so we should ignore the + // next one we get. + if t.state == stopping && ev.StopSignal() != syscall.SIGSTOP { + t.ignoreNextSigstop = true + } + + // TODO(austin) If we're in state stopping and get a SIGSTOP, + // set state stopped instead of stoppedSignal. + + t.setState(state) + + if t.proc.someRunningThread() == nil { + // Nothing is running, uninstall breakpoints + return t.proc.uninstallBreakpoints() + } + // Stop any other running threads + return t.proc.stopAsync() +} + +// onStop adds a handler for state transitions from running to +// non-running states. The handler will be called from the monitor +// thread. +// +// Must be called from the monitor thread. +func (t *thread) onStop(handle func(), onErr func(os.Error)) { + // TODO(austin) This is rather inefficient for things like + // stepping all threads during a continue. Maybe move + // transitionHandlers to the thread, or have both per-thread + // and per-process transition handlers. + h := &transitionHandler{nil, onErr} + h.handle = func(st *thread, old, new threadState) { + if t == st && old.isRunning() && !new.isRunning() { + handle() + } else { + t.proc.transitionHandlers.Push(h) + } + } + t.proc.transitionHandlers.Push(h) +} + +/* + * Event monitor + */ + +// monitor handles debug events and debug requests for p, exiting when +// there are no threads left in p. +func (p *process) monitor() { + var err os.Error + + // Linux requires that all ptrace calls come from the thread + // that originally attached. Prevent the Go scheduler from + // migrating us to other OS threads. + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + hadThreads := false + for err == nil { + p.ready <- true + select { + case event := <-p.debugEvents: + err = event.process() + + case req := <-p.debugReqs: + req.res <- req.f() + + case err = <-p.stopReq: + break + } + + if len(p.threads) == 0 { + if err == nil && hadThreads { + p.logTrace("no more threads; monitor exiting") + err = ProcessExited{} + } + } else { + hadThreads = true + } + } + + // Abort waiting handlers + // TODO(austin) How do I stop the wait threads? + for _, h := range p.transitionHandlers { + h := h.(*transitionHandler) + h.onErr(err) + } + + // Indicate that the monitor cannot receive any more messages + p.err = err + close(p.ready) +} + +// do executes f in the monitor thread (and, thus, atomically with +// respect to thread state changes). f must not block. +// +// Must NOT be called from the monitor thread. +func (p *process) do(f func() os.Error) os.Error { + if !<-p.ready { + return p.err + } + req := &debugReq{f, make(chan os.Error)} + p.debugReqs <- req + return <-req.res +} + +// stopMonitor stops the monitor with the given error. If the monitor +// is already stopped, does nothing. +func (p *process) stopMonitor(err os.Error) { + if err == nil { + panic("cannot stop the monitor with no error") + } + if <-p.ready { + p.stopReq <- err + } +} + +/* + * Public thread interface + */ + +func (t *thread) Regs() (Regs, os.Error) { + var regs syscall.PtraceRegs + + err := t.proc.do(func() os.Error { + if !t.state.isStopped() { + return &badState{t, "cannot get registers", t.state} + } + return t.ptraceGetRegs(®s) + }) + if err != nil { + return nil, err + } + + setter := func(r *syscall.PtraceRegs) os.Error { + return t.proc.do(func() os.Error { + if !t.state.isStopped() { + return &badState{t, "cannot get registers", t.state} + } + return t.ptraceSetRegs(r) + }) + } + return newRegs(®s, setter), nil +} + +func (t *thread) Peek(addr Word, out []byte) (int, os.Error) { + var c int + + err := t.proc.do(func() os.Error { + if !t.state.isStopped() { + return &badState{t, "cannot peek text", t.state} + } + + var err os.Error + c, err = t.ptracePeekText(uintptr(addr), out) + return err + }) + + return c, err +} + +func (t *thread) Poke(addr Word, out []byte) (int, os.Error) { + var c int + + err := t.proc.do(func() os.Error { + if !t.state.isStopped() { + return &badState{t, "cannot poke text", t.state} + } + + var err os.Error + c, err = t.ptracePokeText(uintptr(addr), out) + return err + }) + + return c, err +} + +// stepAsync starts this thread single stepping. When the single step +// is complete, it will send nil on the given channel. If an error +// occurs while setting up the single step, it returns that error. If +// an error occurs while waiting for the single step to complete, it +// sends that error on the channel. +func (t *thread) stepAsync(ready chan os.Error) os.Error { + if err := t.ptraceStep(); err != nil { + return err + } + t.setState(singleStepping) + t.onStop(func() { ready <- nil }, + func(err os.Error) { ready <- err }) + return nil +} + +func (t *thread) Step() os.Error { + t.logTrace("Step {") + defer t.logTrace("}") + + ready := make(chan os.Error) + + err := t.proc.do(func() os.Error { + if !t.state.isStopped() { + return &badState{t, "cannot single step", t.state} + } + return t.stepAsync(ready) + }) + if err != nil { + return err + } + + err = <-ready + return err +} + +// TODO(austin) We should probably get this via C's strsignal. +var sigNames = [...]string{ + "SIGEXIT", "SIGHUP", "SIGINT", "SIGQUIT", "SIGILL", + "SIGTRAP", "SIGABRT", "SIGBUS", "SIGFPE", "SIGKILL", + "SIGUSR1", "SIGSEGV", "SIGUSR2", "SIGPIPE", "SIGALRM", + "SIGTERM", "SIGSTKFLT", "SIGCHLD", "SIGCONT", "SIGSTOP", + "SIGTSTP", "SIGTTIN", "SIGTTOU", "SIGURG", "SIGXCPU", + "SIGXFSZ", "SIGVTALRM", "SIGPROF", "SIGWINCH", "SIGPOLL", + "SIGPWR", "SIGSYS", +} + +// sigName returns the symbolic name for the given signal number. If +// the signal number is invalid, returns "". +func sigName(signal int) string { + if signal < 0 || signal >= len(sigNames) { + return "" + } + return sigNames[signal] +} + +func (t *thread) Stopped() (Cause, os.Error) { + var c Cause + err := t.proc.do(func() os.Error { + switch t.state { + case stopped: + c = Stopped{} + + case stoppedBreakpoint: + c = Breakpoint(t.breakpoint.pc) + + case stoppedSignal: + c = Signal(sigName(t.signal)) + + case stoppedThreadCreate: + c = &ThreadCreate{t.newThread} + + case stoppedExiting, exiting, exited: + if t.signal == -1 { + c = &ThreadExit{t.exitStatus, ""} + } else { + c = &ThreadExit{t.exitStatus, sigName(t.signal)} + } + + default: + return &badState{t, "cannot get stop cause", t.state} + } + return nil + }) + if err != nil { + return nil, err + } + + return c, nil +} + +func (p *process) Threads() []Thread { + var res []Thread + + p.do(func() os.Error { + res = make([]Thread, len(p.threads)) + i := 0 + for _, t := range p.threads { + // Exclude zombie threads. + st := t.state + if st == exiting || st == exited || st == detached { + continue + } + + res[i] = t + i++ + } + res = res[0:i] + return nil + }) + return res +} + +func (p *process) AddBreakpoint(pc Word) os.Error { + return p.do(func() os.Error { + if t := p.someRunningThread(); t != nil { + return &badState{t, "cannot add breakpoint", t.state} + } + if _, ok := p.breakpoints[uintptr(pc)]; ok { + return breakpointExistsError(pc) + } + p.breakpoints[uintptr(pc)] = &breakpoint{pc: uintptr(pc)} + return nil + }) +} + +func (p *process) RemoveBreakpoint(pc Word) os.Error { + return p.do(func() os.Error { + if t := p.someRunningThread(); t != nil { + return &badState{t, "cannot remove breakpoint", t.state} + } + if _, ok := p.breakpoints[uintptr(pc)]; !ok { + return noBreakpointError(pc) + } + p.breakpoints[uintptr(pc)] = nil, false + return nil + }) +} + +func (p *process) Continue() os.Error { + // Single step any threads that are stopped at breakpoints so + // we can reinstall breakpoints. + var ready chan os.Error + count := 0 + + err := p.do(func() os.Error { + // We make the ready channel big enough to hold all + // ready message so we don't jam up the monitor if we + // stop listening (e.g., if there's an error). + ready = make(chan os.Error, len(p.threads)) + + for _, t := range p.threads { + if !t.state.isStopped() { + continue + } + + // We use the breakpoint map directly here + // instead of checking the stop cause because + // it could have been stopped at a breakpoint + // for some other reason, or the breakpoint + // could have been added since it was stopped. + var regs syscall.PtraceRegs + err := t.ptraceGetRegs(®s) + if err != nil { + return err + } + if b, ok := p.breakpoints[uintptr(regs.PC())]; ok { + t.logTrace("stepping over breakpoint %v", b) + if err := t.stepAsync(ready); err != nil { + return err + } + count++ + } + } + return nil + }) + if err != nil { + p.stopMonitor(err) + return err + } + + // Wait for single stepping threads + for count > 0 { + err = <-ready + if err != nil { + p.stopMonitor(err) + return err + } + count-- + } + + // Continue all threads + err = p.do(func() os.Error { + if err := p.installBreakpoints(); err != nil { + return err + } + + for _, t := range p.threads { + var err os.Error + switch { + case !t.state.isStopped(): + continue + + case t.state == stoppedSignal && t.signal != syscall.SIGSTOP && t.signal != syscall.SIGTRAP: + t.logTrace("continuing with signal %d", t.signal) + err = t.ptraceContWithSignal(t.signal) + + default: + t.logTrace("continuing") + err = t.ptraceCont() + } + if err != nil { + return err + } + if t.state == stoppedExiting { + t.setState(exiting) + } else { + t.setState(running) + } + } + return nil + }) + if err != nil { + // TODO(austin) Do we need to stop the monitor with + // this error atomically with the do-routine above? + p.stopMonitor(err) + return err + } + + return nil +} + +func (p *process) WaitStop() os.Error { + // We need a non-blocking ready channel for the case where all + // threads are already stopped. + ready := make(chan os.Error, 1) + + err := p.do(func() os.Error { + // Are all of the threads already stopped? + if p.someRunningThread() == nil { + ready <- nil + return nil + } + + // Monitor state transitions + h := &transitionHandler{} + h.handle = func(st *thread, old, new threadState) { + if !new.isRunning() { + if p.someRunningThread() == nil { + ready <- nil + return + } + } + p.transitionHandlers.Push(h) + } + h.onErr = func(err os.Error) { ready <- err } + p.transitionHandlers.Push(h) + return nil + }) + if err != nil { + return err + } + + return <-ready +} + +func (p *process) Stop() os.Error { + err := p.do(func() os.Error { return p.stopAsync() }) + if err != nil { + return err + } + + return p.WaitStop() +} + +func (p *process) Detach() os.Error { + if err := p.Stop(); err != nil { + return err + } + + err := p.do(func() os.Error { + if err := p.uninstallBreakpoints(); err != nil { + return err + } + + for pid, t := range p.threads { + if t.state.isStopped() { + // We can't detach from zombies. + if err := t.ptraceDetach(); err != nil { + return err + } + } + t.setState(detached) + p.threads[pid] = nil, false + } + return nil + }) + // TODO(austin) Wait for monitor thread to exit? + return err +} + +// newThread creates a new thread object and waits for its initial +// signal. If cloned is true, this thread was cloned from a thread we +// are already attached to. +// +// Must be run from the monitor thread. +func (p *process) newThread(tid int, signal int, cloned bool) (*thread, os.Error) { + t := &thread{tid: tid, proc: p, state: stopped} + + // Get the signal from the thread + // TODO(austin) Thread might already be stopped if we're attaching. + w, err := os.Wait(tid, syscall.WALL) + if err != nil { + return nil, err + } + if w.Pid != tid || w.StopSignal() != signal { + return nil, &newThreadError{w, tid, signal} + } + + if !cloned { + err = t.ptraceSetOptions(syscall.PTRACE_O_TRACECLONE | syscall.PTRACE_O_TRACEEXIT) + if err != nil { + return nil, err + } + } + + p.threads[tid] = t + + return t, nil +} + +// attachThread attaches a running thread to the process. +// +// Must NOT be run from the monitor thread. +func (p *process) attachThread(tid int) (*thread, os.Error) { + p.logTrace("attaching to thread %d", tid) + var thr *thread + err := p.do(func() os.Error { + errno := syscall.PtraceAttach(tid) + if errno != 0 { + return os.NewSyscallError("ptrace(ATTACH)", errno) + } + + var err os.Error + thr, err = p.newThread(tid, syscall.SIGSTOP, false) + return err + }) + return thr, err +} + +// attachAllThreads attaches to all threads in a process. +func (p *process) attachAllThreads() os.Error { + taskPath := "/proc/" + strconv.Itoa(p.pid) + "/task" + taskDir, err := os.Open(taskPath, os.O_RDONLY, 0) + if err != nil { + return err + } + defer taskDir.Close() + + // We stop threads as we attach to them; however, because new + // threads can appear while we're looping over all of them, we + // have to repeatly scan until we know we're attached to all + // of them. + for again := true; again; { + again = false + + tids, err := taskDir.Readdirnames(-1) + if err != nil { + return err + } + + for _, tidStr := range tids { + tid, err := strconv.Atoi(tidStr) + if err != nil { + return err + } + if _, ok := p.threads[tid]; ok { + continue + } + + _, err = p.attachThread(tid) + if err != nil { + // There could have been a race, or + // this process could be a zobmie. + statFile, err2 := ioutil.ReadFile(taskPath + "/" + tidStr + "/stat") + if err2 != nil { + switch err2 := err2.(type) { + case *os.PathError: + if err2.Error == os.ENOENT { + // Raced with thread exit + p.logTrace("raced with thread %d exit", tid) + continue + } + } + // Return the original error + return err + } + + statParts := strings.Split(string(statFile), " ", 4) + if len(statParts) > 2 && statParts[2] == "Z" { + // tid is a zombie + p.logTrace("thread %d is a zombie", tid) + continue + } + + // Return the original error + return err + } + again = true + } + } + + return nil +} + +// newProcess creates a new process object and starts its monitor thread. +func newProcess(pid int) *process { + p := &process{ + pid: pid, + threads: make(map[int]*thread), + breakpoints: make(map[uintptr]*breakpoint), + ready: make(chan bool, 1), + debugEvents: make(chan *debugEvent), + debugReqs: make(chan *debugReq), + stopReq: make(chan os.Error), + } + + go p.monitor() + + return p +} + +// Attach attaches to process pid and stops all of its threads. +func Attach(pid int) (Process, os.Error) { + p := newProcess(pid) + + // Attach to all threads + err := p.attachAllThreads() + if err != nil { + p.Detach() + // TODO(austin) Detach stopped the monitor already + //p.stopMonitor(err); + return nil, err + } + + return p, nil +} + +// ForkExec forks the current process and execs argv0, stopping the +// new process after the exec syscall. See os.ForkExec for additional +// details. +func ForkExec(argv0 string, argv []string, envv []string, dir string, fd []*os.File) (Process, os.Error) { + p := newProcess(-1) + + // Create array of integer (system) fds. + intfd := make([]int, len(fd)) + for i, f := range fd { + if f == nil { + intfd[i] = -1 + } else { + intfd[i] = f.Fd() + } + } + + // Fork from the monitor thread so we get the right tracer pid. + err := p.do(func() os.Error { + pid, errno := syscall.PtraceForkExec(argv0, argv, envv, dir, intfd) + if errno != 0 { + return &os.PathError{"fork/exec", argv0, os.Errno(errno)} + } + p.pid = pid + + // The process will raise SIGTRAP when it reaches execve. + _, err := p.newThread(pid, syscall.SIGTRAP, false) + return err + }) + if err != nil { + p.stopMonitor(err) + return nil, err + } + + return p, nil +} diff --git a/libgo/go/debug/proc/proc_rtems.go b/libgo/go/debug/proc/proc_rtems.go new file mode 100644 index 000000000..5311a63ba --- /dev/null +++ b/libgo/go/debug/proc/proc_rtems.go @@ -0,0 +1,17 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package proc + +import "os" + +// Process tracing is not supported on RTEMS yet. + +func Attach(pid int) (Process, os.Error) { + return nil, os.NewError("debug/proc not implemented on RTEMS") +} + +func ForkExec(argv0 string, argv []string, envv []string, dir string, fd []*os.File) (Process, os.Error) { + return Attach(0) +} diff --git a/libgo/go/debug/proc/proc_solaris.go b/libgo/go/debug/proc/proc_solaris.go new file mode 100644 index 000000000..a72c59237 --- /dev/null +++ b/libgo/go/debug/proc/proc_solaris.go @@ -0,0 +1,17 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package proc + +import "os" + +// Process tracing is not supported on Solaris yet. + +func Attach(pid int) (Process, os.Error) { + return nil, os.NewError("debug/proc not implemented on Solaris") +} + +func ForkExec(argv0 string, argv []string, envv []string, dir string, fd []*os.File) (Process, os.Error) { + return Attach(0) +} diff --git a/libgo/go/debug/proc/proc_windows.go b/libgo/go/debug/proc/proc_windows.go new file mode 100644 index 000000000..dc22faef8 --- /dev/null +++ b/libgo/go/debug/proc/proc_windows.go @@ -0,0 +1,17 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package proc + +import "os" + +// Process tracing is not supported on windows yet. + +func Attach(pid int) (Process, os.Error) { + return nil, os.NewError("debug/proc not implemented on windows") +} + +func ForkExec(argv0 string, argv []string, envv []string, dir string, fd []*os.File) (Process, os.Error) { + return Attach(0) +} diff --git a/libgo/go/debug/proc/ptrace-nptl.txt b/libgo/go/debug/proc/ptrace-nptl.txt new file mode 100644 index 000000000..62cbf7700 --- /dev/null +++ b/libgo/go/debug/proc/ptrace-nptl.txt @@ -0,0 +1,132 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +ptrace and NTPL, the missing manpage + +== Signals == + +A signal sent to a ptrace'd process or thread causes only the thread +that receives it to stop and report to the attached process. + +Use tgkill to target a signal (for example, SIGSTOP) at a particular +thread. If you use kill, the signal could be delivered to another +thread in the same process. + +Note that SIGSTOP differs from its usual behavior when a process is +being traced. Usually, a SIGSTOP sent to any thread in a thread group +will stop all threads in the thread group. When a thread is traced, +however, a SIGSTOP affects only the receiving thread (and any other +threads in the thread group that are not traced). + +SIGKILL behaves like it does for non-traced processes. It affects all +threads in the process and terminates them without the WSTOPSIG event +generated by other signals. However, if PTRACE_O_TRACEEXIT is set, +the attached process will still receive PTRACE_EVENT_EXIT events +before receiving WIFSIGNALED events. + +See "Following thread death" for a caveat regarding signal delivery to +zombie threads. + +== Waiting on threads == + +Cloned threads in ptrace'd processes are treated similarly to cloned +threads in your own process. Thus, you must use the __WALL option in +order to receive notifications from threads created by the child +process. Similarly, the __WCLONE option will wait only on +notifications from threads created by the child process and *not* on +notifications from the initial child thread. + +Even when waiting on a specific thread's PID using waitpid or similar, +__WALL or __WCLONE is necessary or waitpid will return ECHILD. + +== Attaching to existing threads == + +libthread_db (which gdb uses), attaches to existing threads by pulling +the pthread data structures out of the traced process. The much +easier way is to traverse the /proc/PID/task directory, though it's +unclear how the semantics of these two approaches differ. + +Unfortunately, if the main thread has exited (but the overall process +has not), it sticks around as a zombie process. This zombie will +appear in the /proc/PID/task directory, but trying to attach to it +will yield EPERM. In this case, the third field of the +/proc/PID/task/PID/stat file will be "Z". Attempting to open the stat +file is also a convenient way to detect races between listing the task +directory and the thread exiting. Coincidentally, gdb will simply +fail to attach to a process whose main thread is a zombie. + +Because new threads may be created while the debugger is in the +process of attaching to existing threads, the debugger must repeatedly +re-list the task directory until it has attached to (and thus stopped) +every thread listed. + +In order to follow new threads created by existing threads, +PTRACE_O_TRACECLONE must be set on each thread attached to. + +== Following new threads == + +With the child process stopped, use PTRACE_SETOPTIONS to set the +PTRACE_O_TRACECLONE option. This option is per-thread, and thus must +be set on each existing thread individually. When an existing thread +with PTRACE_O_TRACECLONE set spawns a new thread, the existing thread +will stop with (SIGTRAP | PTRACE_EVENT_CLONE << 8) and the PID of the +new thread can be retrieved with PTRACE_GETEVENTMSG on the creating +thread. At this time, the new thread will exist, but will initially +be stopped with a SIGSTOP. The new thread will automatically be +traced and will inherit the PTRACE_O_TRACECLONE option from its +parent. The attached process should wait on the new thread to receive +the SIGSTOP notification. + +When using waitpid(-1, ...), don't rely on the parent thread reporting +a SIGTRAP before receiving the SIGSTOP from the new child thread. + +Without PTRACE_O_TRACECLONE, newly cloned threads will not be +ptrace'd. As a result, signals received by new threads will be +handled in the usual way, which may affect the parent and in turn +appear to the attached process, but attributed to the parent (possibly +in unexpected ways). + +== Following thread death == + +If any thread with the PTRACE_O_TRACEEXIT option set exits (either by +returning or pthread_exit'ing), the tracing process will receive an +immediate PTRACE_EVENT_EXIT. At this point, the thread will still +exist. The exit status, encoded as for wait, can be queried using +PTRACE_GETEVENTMSG on the exiting thread's PID. The thread should be +continued so it can actually exit, after which its wait behavior is +the same as for a thread without the PTRACE_O_TRACEEXIT option. + +If a non-main thread exits (either by returning or pthread_exit'ing), +its corresponding process will also exit, producing a WIFEXITED event +(after the process is continued from a possible PTRACE_EVENT_EXIT +event). It is *not* necessary for another thread to ptrace_join for +this to happen. + +If the main thread exits by returning, then all threads will exit, +first generating a PTRACE_EVENT_EXIT event for each thread if +appropriate, then producing a WIFEXITED event for each thread. + +If the main thread exits using pthread_exit, then it enters a +non-waitable zombie state. It will still produce an immediate +PTRACE_O_TRACEEXIT event, but the WIFEXITED event will be delayed +until the entire process exits. This state exists so that shells +don't think the process is done until all of the threads have exited. +Unfortunately, signals cannot be delivered to non-waitable zombies. +Most notably, SIGSTOP cannot be delivered; as a result, when you +broadcast SIGSTOP to all of the threads, you must not wait for +non-waitable zombies to stop. Furthermore, any ptrace command on a +non-waitable zombie, including PTRACE_DETACH, will return ESRCH. + +== Multi-threaded debuggers == + +If the debugger itself is multi-threaded, ptrace calls must come from +the same thread that originally attached to the remote thread. The +kernel simply compares the PID of the caller of ptrace against the +tracer PID of the process passed to ptrace. Because each debugger +thread has a different PID, calling ptrace from a different thread +might as well be calling it from a different process and the kernel +will return ESRCH. + +wait, on the other hand, does not have this restriction. Any debugger +thread can wait on any thread in the attached process. diff --git a/libgo/go/debug/proc/regs_darwin_386.go b/libgo/go/debug/proc/regs_darwin_386.go new file mode 100644 index 000000000..60c9ac719 --- /dev/null +++ b/libgo/go/debug/proc/regs_darwin_386.go @@ -0,0 +1,5 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package proc diff --git a/libgo/go/debug/proc/regs_darwin_amd64.go b/libgo/go/debug/proc/regs_darwin_amd64.go new file mode 100644 index 000000000..60c9ac719 --- /dev/null +++ b/libgo/go/debug/proc/regs_darwin_amd64.go @@ -0,0 +1,5 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package proc diff --git a/libgo/go/debug/proc/regs_freebsd_386.go b/libgo/go/debug/proc/regs_freebsd_386.go new file mode 100644 index 000000000..60c9ac719 --- /dev/null +++ b/libgo/go/debug/proc/regs_freebsd_386.go @@ -0,0 +1,5 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package proc diff --git a/libgo/go/debug/proc/regs_freebsd_amd64.go b/libgo/go/debug/proc/regs_freebsd_amd64.go new file mode 100644 index 000000000..60c9ac719 --- /dev/null +++ b/libgo/go/debug/proc/regs_freebsd_amd64.go @@ -0,0 +1,5 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package proc diff --git a/libgo/go/debug/proc/regs_linux_386.go b/libgo/go/debug/proc/regs_linux_386.go new file mode 100644 index 000000000..b4a9769db --- /dev/null +++ b/libgo/go/debug/proc/regs_linux_386.go @@ -0,0 +1,143 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package proc + +import ( + "os" + "strconv" + "syscall" +) + +type _386Regs struct { + syscall.PtraceRegs + setter func(*syscall.PtraceRegs) os.Error +} + +var names = []string{ + "eax", + "ebx", + "ecx", + "edx", + "esi", + "edi", + "ebp", + "esp", + "eip", + "eflags", + "cs", + "ss", + "ds", + "es", + "fs", + "gs", +} + +func (r *_386Regs) PC() Word { return Word(r.Eip) } + +func (r *_386Regs) SetPC(val Word) os.Error { + r.Eip = int32(val) + return r.setter(&r.PtraceRegs) +} + +func (r *_386Regs) Link() Word { + // TODO(austin) + panic("No link register") +} + +func (r *_386Regs) SetLink(val Word) os.Error { panic("No link register") } + +func (r *_386Regs) SP() Word { return Word(r.Esp) } + +func (r *_386Regs) SetSP(val Word) os.Error { + r.Esp = int32(val) + return r.setter(&r.PtraceRegs) +} + +func (r *_386Regs) Names() []string { return names } + +func (r *_386Regs) Get(i int) Word { + switch i { + case 0: + return Word(uint32(r.Eax)) + case 1: + return Word(uint32(r.Ebx)) + case 2: + return Word(uint32(r.Ecx)) + case 3: + return Word(uint32(r.Edx)) + case 4: + return Word(uint32(r.Esi)) + case 5: + return Word(uint32(r.Edi)) + case 6: + return Word(uint32(r.Ebp)) + case 7: + return Word(uint32(r.Esp)) + case 8: + return Word(uint32(r.Eip)) + case 9: + return Word(uint32(r.Eflags)) + case 10: + return Word(r.Xcs) + case 11: + return Word(r.Xss) + case 12: + return Word(r.Xds) + case 13: + return Word(r.Xes) + case 14: + return Word(r.Xfs) + case 15: + return Word(r.Xgs) + } + panic("invalid register index " + strconv.Itoa(i)) +} + +func (r *_386Regs) Set(i int, val Word) os.Error { + switch i { + case 0: + r.Eax = int32(val) + case 1: + r.Ebx = int32(val) + case 2: + r.Ecx = int32(val) + case 3: + r.Edx = int32(val) + case 4: + r.Esi = int32(val) + case 5: + r.Edi = int32(val) + case 6: + r.Ebp = int32(val) + case 7: + r.Esp = int32(val) + case 8: + r.Eip = int32(val) + case 9: + r.Eflags = int32(val) + case 10: + r.Xcs = int32(val) + case 11: + r.Xss = int32(val) + case 12: + r.Xds = int32(val) + case 13: + r.Xes = int32(val) + case 14: + r.Xfs = int32(val) + case 15: + r.Xgs = int32(val) + default: + panic("invalid register index " + strconv.Itoa(i)) + } + return r.setter(&r.PtraceRegs) +} + +func newRegs(regs *syscall.PtraceRegs, setter func(*syscall.PtraceRegs) os.Error) Regs { + res := _386Regs{} + res.PtraceRegs = *regs + res.setter = setter + return &res +} diff --git a/libgo/go/debug/proc/regs_linux_amd64.go b/libgo/go/debug/proc/regs_linux_amd64.go new file mode 100644 index 000000000..381be29b1 --- /dev/null +++ b/libgo/go/debug/proc/regs_linux_amd64.go @@ -0,0 +1,191 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package proc + +import ( + "os" + "strconv" + "syscall" +) + +type amd64Regs struct { + syscall.PtraceRegs + setter func(*syscall.PtraceRegs) os.Error +} + +var names = [...]string{ + "rax", + "rbx", + "rcx", + "rdx", + "rsi", + "rdi", + "rbp", + "rsp", + "r8", + "r9", + "r10", + "r11", + "r12", + "r13", + "r14", + "r15", + "rip", + "eflags", + "cs", + "ss", + "ds", + "es", + "fs", + "gs", + + // PtraceRegs contains these registers, but I don't think + // they're actually meaningful. + //"orig_rax", + //"fs_base", + //"gs_base", +} + +func (r *amd64Regs) PC() Word { return Word(r.Rip) } + +func (r *amd64Regs) SetPC(val Word) os.Error { + r.Rip = uint64(val) + return r.setter(&r.PtraceRegs) +} + +func (r *amd64Regs) Link() Word { + // TODO(austin) + panic("No link register") +} + +func (r *amd64Regs) SetLink(val Word) os.Error { + panic("No link register") +} + +func (r *amd64Regs) SP() Word { return Word(r.Rsp) } + +func (r *amd64Regs) SetSP(val Word) os.Error { + r.Rsp = uint64(val) + return r.setter(&r.PtraceRegs) +} + +func (r *amd64Regs) Names() []string { return names[0:] } + +func (r *amd64Regs) Get(i int) Word { + switch i { + case 0: + return Word(r.Rax) + case 1: + return Word(r.Rbx) + case 2: + return Word(r.Rcx) + case 3: + return Word(r.Rdx) + case 4: + return Word(r.Rsi) + case 5: + return Word(r.Rdi) + case 6: + return Word(r.Rbp) + case 7: + return Word(r.Rsp) + case 8: + return Word(r.R8) + case 9: + return Word(r.R9) + case 10: + return Word(r.R10) + case 11: + return Word(r.R11) + case 12: + return Word(r.R12) + case 13: + return Word(r.R13) + case 14: + return Word(r.R14) + case 15: + return Word(r.R15) + case 16: + return Word(r.Rip) + case 17: + return Word(r.Eflags) + case 18: + return Word(r.Cs) + case 19: + return Word(r.Ss) + case 20: + return Word(r.Ds) + case 21: + return Word(r.Es) + case 22: + return Word(r.Fs) + case 23: + return Word(r.Gs) + } + panic("invalid register index " + strconv.Itoa(i)) +} + +func (r *amd64Regs) Set(i int, val Word) os.Error { + switch i { + case 0: + r.Rax = uint64(val) + case 1: + r.Rbx = uint64(val) + case 2: + r.Rcx = uint64(val) + case 3: + r.Rdx = uint64(val) + case 4: + r.Rsi = uint64(val) + case 5: + r.Rdi = uint64(val) + case 6: + r.Rbp = uint64(val) + case 7: + r.Rsp = uint64(val) + case 8: + r.R8 = uint64(val) + case 9: + r.R9 = uint64(val) + case 10: + r.R10 = uint64(val) + case 11: + r.R11 = uint64(val) + case 12: + r.R12 = uint64(val) + case 13: + r.R13 = uint64(val) + case 14: + r.R14 = uint64(val) + case 15: + r.R15 = uint64(val) + case 16: + r.Rip = uint64(val) + case 17: + r.Eflags = uint64(val) + case 18: + r.Cs = uint64(val) + case 19: + r.Ss = uint64(val) + case 20: + r.Ds = uint64(val) + case 21: + r.Es = uint64(val) + case 22: + r.Fs = uint64(val) + case 23: + r.Gs = uint64(val) + default: + panic("invalid register index " + strconv.Itoa(i)) + } + return r.setter(&r.PtraceRegs) +} + +func newRegs(regs *syscall.PtraceRegs, setter func(*syscall.PtraceRegs) os.Error) Regs { + res := amd64Regs{} + res.PtraceRegs = *regs + res.setter = setter + return &res +} diff --git a/libgo/go/debug/proc/regs_linux_arm.go b/libgo/go/debug/proc/regs_linux_arm.go new file mode 100644 index 000000000..ec78cbcf2 --- /dev/null +++ b/libgo/go/debug/proc/regs_linux_arm.go @@ -0,0 +1,39 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package proc + +import ( + "os" + "syscall" +) + +// TODO(kaib): add support + +type armRegs struct{} + +func (r *armRegs) PC() Word { return Word(0) } + +func (r *armRegs) SetPC(val Word) os.Error { return nil } + +func (r *armRegs) Link() Word { return Word(0) } + +func (r *armRegs) SetLink(val Word) os.Error { return nil } + +func (r *armRegs) SP() Word { return Word(0) } + +func (r *armRegs) SetSP(val Word) os.Error { return nil } + +func (r *armRegs) Names() []string { return nil } + +func (r *armRegs) Get(i int) Word { return Word(0) } + +func (r *armRegs) Set(i int, val Word) os.Error { + return nil +} + +func newRegs(regs *syscall.PtraceRegs, setter func(*syscall.PtraceRegs) os.Error) Regs { + res := armRegs{} + return &res +} diff --git a/libgo/go/debug/proc/regs_windows_386.go b/libgo/go/debug/proc/regs_windows_386.go new file mode 100644 index 000000000..60c9ac719 --- /dev/null +++ b/libgo/go/debug/proc/regs_windows_386.go @@ -0,0 +1,5 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package proc diff --git a/libgo/go/debug/proc/regs_windows_amd64.go b/libgo/go/debug/proc/regs_windows_amd64.go new file mode 100644 index 000000000..60c9ac719 --- /dev/null +++ b/libgo/go/debug/proc/regs_windows_amd64.go @@ -0,0 +1,5 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package proc diff --git a/libgo/go/ebnf/ebnf.go b/libgo/go/ebnf/ebnf.go new file mode 100644 index 000000000..e5aabd582 --- /dev/null +++ b/libgo/go/ebnf/ebnf.go @@ -0,0 +1,248 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// A library for EBNF grammars. The input is text ([]byte) satisfying +// the following grammar (represented itself in EBNF): +// +// Production = name "=" Expression "." . +// Expression = Alternative { "|" Alternative } . +// Alternative = Term { Term } . +// Term = name | token [ "..." token ] | Group | Option | Repetition . +// Group = "(" Expression ")" . +// Option = "[" Expression "]" . +// Repetition = "{" Expression "}" . +// +// A name is a Go identifier, a token is a Go string, and comments +// and white space follow the same rules as for the Go language. +// Production names starting with an uppercase Unicode letter denote +// non-terminal productions (i.e., productions which allow white-space +// and comments between tokens); all other production names denote +// lexical productions. +// +package ebnf + +import ( + "go/scanner" + "go/token" + "os" + "unicode" + "utf8" +) + + +// ---------------------------------------------------------------------------- +// Internal representation + +type ( + // An Expression node represents a production expression. + Expression interface { + // Pos is the position of the first character of the syntactic construct + Pos() token.Pos + } + + // An Alternative node represents a non-empty list of alternative expressions. + Alternative []Expression // x | y | z + + // A Sequence node represents a non-empty list of sequential expressions. + Sequence []Expression // x y z + + // A Name node represents a production name. + Name struct { + StringPos token.Pos + String string + } + + // A Token node represents a literal. + Token struct { + StringPos token.Pos + String string + } + + // A List node represents a range of characters. + Range struct { + Begin, End *Token // begin ... end + } + + // A Group node represents a grouped expression. + Group struct { + Lparen token.Pos + Body Expression // (body) + } + + // An Option node represents an optional expression. + Option struct { + Lbrack token.Pos + Body Expression // [body] + } + + // A Repetition node represents a repeated expression. + Repetition struct { + Lbrace token.Pos + Body Expression // {body} + } + + // A Production node represents an EBNF production. + Production struct { + Name *Name + Expr Expression + } + + // A Grammar is a set of EBNF productions. The map + // is indexed by production name. + // + Grammar map[string]*Production +) + + +func (x Alternative) Pos() token.Pos { return x[0].Pos() } // the parser always generates non-empty Alternative +func (x Sequence) Pos() token.Pos { return x[0].Pos() } // the parser always generates non-empty Sequences +func (x *Name) Pos() token.Pos { return x.StringPos } +func (x *Token) Pos() token.Pos { return x.StringPos } +func (x *Range) Pos() token.Pos { return x.Begin.Pos() } +func (x *Group) Pos() token.Pos { return x.Lparen } +func (x *Option) Pos() token.Pos { return x.Lbrack } +func (x *Repetition) Pos() token.Pos { return x.Lbrace } +func (x *Production) Pos() token.Pos { return x.Name.Pos() } + + +// ---------------------------------------------------------------------------- +// Grammar verification + +func isLexical(name string) bool { + ch, _ := utf8.DecodeRuneInString(name) + return !unicode.IsUpper(ch) +} + + +type verifier struct { + fset *token.FileSet + scanner.ErrorVector + worklist []*Production + reached Grammar // set of productions reached from (and including) the root production + grammar Grammar +} + + +func (v *verifier) error(pos token.Pos, msg string) { + v.Error(v.fset.Position(pos), msg) +} + + +func (v *verifier) push(prod *Production) { + name := prod.Name.String + if _, found := v.reached[name]; !found { + v.worklist = append(v.worklist, prod) + v.reached[name] = prod + } +} + + +func (v *verifier) verifyChar(x *Token) int { + s := x.String + if utf8.RuneCountInString(s) != 1 { + v.error(x.Pos(), "single char expected, found "+s) + return 0 + } + ch, _ := utf8.DecodeRuneInString(s) + return ch +} + + +func (v *verifier) verifyExpr(expr Expression, lexical bool) { + switch x := expr.(type) { + case nil: + // empty expression + case Alternative: + for _, e := range x { + v.verifyExpr(e, lexical) + } + case Sequence: + for _, e := range x { + v.verifyExpr(e, lexical) + } + case *Name: + // a production with this name must exist; + // add it to the worklist if not yet processed + if prod, found := v.grammar[x.String]; found { + v.push(prod) + } else { + v.error(x.Pos(), "missing production "+x.String) + } + // within a lexical production references + // to non-lexical productions are invalid + if lexical && !isLexical(x.String) { + v.error(x.Pos(), "reference to non-lexical production "+x.String) + } + case *Token: + // nothing to do for now + case *Range: + i := v.verifyChar(x.Begin) + j := v.verifyChar(x.End) + if i >= j { + v.error(x.Pos(), "decreasing character range") + } + case *Group: + v.verifyExpr(x.Body, lexical) + case *Option: + v.verifyExpr(x.Body, lexical) + case *Repetition: + v.verifyExpr(x.Body, lexical) + default: + panic("unreachable") + } +} + + +func (v *verifier) verify(fset *token.FileSet, grammar Grammar, start string) { + // find root production + root, found := grammar[start] + if !found { + // token.NoPos doesn't require a file set; + // ok to set v.fset only afterwards + v.error(token.NoPos, "no start production "+start) + return + } + + // initialize verifier + v.fset = fset + v.ErrorVector.Reset() + v.worklist = v.worklist[0:0] + v.reached = make(Grammar) + v.grammar = grammar + + // work through the worklist + v.push(root) + for { + n := len(v.worklist) - 1 + if n < 0 { + break + } + prod := v.worklist[n] + v.worklist = v.worklist[0:n] + v.verifyExpr(prod.Expr, isLexical(prod.Name.String)) + } + + // check if all productions were reached + if len(v.reached) < len(v.grammar) { + for name, prod := range v.grammar { + if _, found := v.reached[name]; !found { + v.error(prod.Pos(), name+" is unreachable") + } + } + } +} + + +// Verify checks that: +// - all productions used are defined +// - all productions defined are used when beginning at start +// - lexical productions refer only to other lexical productions +// +// Position information is interpreted relative to the file set fset. +// +func Verify(fset *token.FileSet, grammar Grammar, start string) os.Error { + var v verifier + v.verify(fset, grammar, start) + return v.GetError(scanner.Sorted) +} diff --git a/libgo/go/ebnf/ebnf_test.go b/libgo/go/ebnf/ebnf_test.go new file mode 100644 index 000000000..bbe530c27 --- /dev/null +++ b/libgo/go/ebnf/ebnf_test.go @@ -0,0 +1,77 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ebnf + +import ( + "go/token" + "io/ioutil" + "testing" +) + + +var fset = token.NewFileSet() + + +var grammars = []string{ + `Program = . + `, + + `Program = foo . + foo = "foo" . + `, + + `Program = "a" | "b" "c" . + `, + + `Program = "a" ... "z" . + `, + + `Program = Song . + Song = { Note } . + Note = Do | (Re | Mi | Fa | So | La) | Ti . + Do = "c" . + Re = "d" . + Mi = "e" . + Fa = "f" . + So = "g" . + La = "a" . + Ti = ti . + ti = "b" . + `, +} + + +func check(t *testing.T, filename string, src []byte) { + grammar, err := Parse(fset, filename, src) + if err != nil { + t.Errorf("Parse(%s) failed: %v", src, err) + } + if err = Verify(fset, grammar, "Program"); err != nil { + t.Errorf("Verify(%s) failed: %v", src, err) + } +} + + +func TestGrammars(t *testing.T) { + for _, src := range grammars { + check(t, "", []byte(src)) + } +} + + +var files = []string{ +// TODO(gri) add some test files +} + + +func TestFiles(t *testing.T) { + for _, filename := range files { + src, err := ioutil.ReadFile(filename) + if err != nil { + t.Fatal(err) + } + check(t, filename, src) + } +} diff --git a/libgo/go/ebnf/parser.go b/libgo/go/ebnf/parser.go new file mode 100644 index 000000000..c38530177 --- /dev/null +++ b/libgo/go/ebnf/parser.go @@ -0,0 +1,208 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ebnf + +import ( + "go/scanner" + "go/token" + "os" + "strconv" +) + + +type parser struct { + fset *token.FileSet + scanner.ErrorVector + scanner scanner.Scanner + pos token.Pos // token position + tok token.Token // one token look-ahead + lit []byte // token literal +} + + +func (p *parser) next() { + p.pos, p.tok, p.lit = p.scanner.Scan() + if p.tok.IsKeyword() { + // TODO Should keyword mapping always happen outside scanner? + // Or should there be a flag to scanner to enable keyword mapping? + p.tok = token.IDENT + } +} + + +func (p *parser) error(pos token.Pos, msg string) { + p.Error(p.fset.Position(pos), msg) +} + + +func (p *parser) errorExpected(pos token.Pos, msg string) { + msg = "expected " + msg + if pos == p.pos { + // the error happened at the current position; + // make the error message more specific + msg += ", found '" + p.tok.String() + "'" + if p.tok.IsLiteral() { + msg += " " + string(p.lit) + } + } + p.error(pos, msg) +} + + +func (p *parser) expect(tok token.Token) token.Pos { + pos := p.pos + if p.tok != tok { + p.errorExpected(pos, "'"+tok.String()+"'") + } + p.next() // make progress in any case + return pos +} + + +func (p *parser) parseIdentifier() *Name { + pos := p.pos + name := string(p.lit) + p.expect(token.IDENT) + return &Name{pos, name} +} + + +func (p *parser) parseToken() *Token { + pos := p.pos + value := "" + if p.tok == token.STRING { + value, _ = strconv.Unquote(string(p.lit)) + // Unquote may fail with an error, but only if the scanner found + // an illegal string in the first place. In this case the error + // has already been reported. + p.next() + } else { + p.expect(token.STRING) + } + return &Token{pos, value} +} + + +func (p *parser) parseTerm() (x Expression) { + pos := p.pos + + switch p.tok { + case token.IDENT: + x = p.parseIdentifier() + + case token.STRING: + tok := p.parseToken() + x = tok + if p.tok == token.ELLIPSIS { + p.next() + x = &Range{tok, p.parseToken()} + } + + case token.LPAREN: + p.next() + x = &Group{pos, p.parseExpression()} + p.expect(token.RPAREN) + + case token.LBRACK: + p.next() + x = &Option{pos, p.parseExpression()} + p.expect(token.RBRACK) + + case token.LBRACE: + p.next() + x = &Repetition{pos, p.parseExpression()} + p.expect(token.RBRACE) + } + + return x +} + + +func (p *parser) parseSequence() Expression { + var list Sequence + + for x := p.parseTerm(); x != nil; x = p.parseTerm() { + list = append(list, x) + } + + // no need for a sequence if list.Len() < 2 + switch len(list) { + case 0: + return nil + case 1: + return list[0] + } + + return list +} + + +func (p *parser) parseExpression() Expression { + var list Alternative + + for { + if x := p.parseSequence(); x != nil { + list = append(list, x) + } + if p.tok != token.OR { + break + } + p.next() + } + + // no need for an Alternative node if list.Len() < 2 + switch len(list) { + case 0: + return nil + case 1: + return list[0] + } + + return list +} + + +func (p *parser) parseProduction() *Production { + name := p.parseIdentifier() + p.expect(token.ASSIGN) + expr := p.parseExpression() + p.expect(token.PERIOD) + return &Production{name, expr} +} + + +func (p *parser) parse(fset *token.FileSet, filename string, src []byte) Grammar { + // initialize parser + p.fset = fset + p.ErrorVector.Reset() + p.scanner.Init(fset.AddFile(filename, fset.Base(), len(src)), src, p, 0) + p.next() // initializes pos, tok, lit + + grammar := make(Grammar) + for p.tok != token.EOF { + prod := p.parseProduction() + name := prod.Name.String + if _, found := grammar[name]; !found { + grammar[name] = prod + } else { + p.error(prod.Pos(), name+" declared already") + } + } + + return grammar +} + + +// Parse parses a set of EBNF productions from source src. +// It returns a set of productions. Errors are reported +// for incorrect syntax and if a production is declared +// more than once. Position information is recorded relative +// to the file set fset. +// +func Parse(fset *token.FileSet, filename string, src []byte) (Grammar, os.Error) { + var p parser + grammar := p.parse(fset, filename, src) + return grammar, p.GetError(scanner.Sorted) +} diff --git a/libgo/go/encoding/ascii85/ascii85.go b/libgo/go/encoding/ascii85/ascii85.go new file mode 100644 index 000000000..ead0c2475 --- /dev/null +++ b/libgo/go/encoding/ascii85/ascii85.go @@ -0,0 +1,300 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package ascii85 implements the ascii85 data encoding +// as used in the btoa tool and Adobe's PostScript and PDF document formats. +package ascii85 + +import ( + "io" + "os" + "strconv" +) + +/* + * Encoder + */ + +// Encode encodes src into at most MaxEncodedLen(len(src)) +// bytes of dst, returning the actual number of bytes written. +// +// The encoding handles 4-byte chunks, using a special encoding +// for the last fragment, so Encode is not appropriate for use on +// individual blocks of a large data stream. Use NewEncoder() instead. +// +// Often, ascii85-encoded data is wrapped in <~ and ~> symbols. +// Encode does not add these. +func Encode(dst, src []byte) int { + if len(src) == 0 { + return 0 + } + + n := 0 + for len(src) > 0 { + dst[0] = 0 + dst[1] = 0 + dst[2] = 0 + dst[3] = 0 + dst[4] = 0 + + // Unpack 4 bytes into uint32 to repack into base 85 5-byte. + var v uint32 + switch len(src) { + default: + v |= uint32(src[3]) + fallthrough + case 3: + v |= uint32(src[2]) << 8 + fallthrough + case 2: + v |= uint32(src[1]) << 16 + fallthrough + case 1: + v |= uint32(src[0]) << 24 + } + + // Special case: zero (!!!!!) shortens to z. + if v == 0 && len(src) >= 4 { + dst[0] = 'z' + dst = dst[1:] + n++ + continue + } + + // Otherwise, 5 base 85 digits starting at !. + for i := 4; i >= 0; i-- { + dst[i] = '!' + byte(v%85) + v /= 85 + } + + // If src was short, discard the low destination bytes. + m := 5 + if len(src) < 4 { + m -= 4 - len(src) + src = nil + } else { + src = src[4:] + } + dst = dst[m:] + n += m + } + return n +} + +// MaxEncodedLen returns the maximum length of an encoding of n source bytes. +func MaxEncodedLen(n int) int { return (n + 3) / 4 * 5 } + +// NewEncoder returns a new ascii85 stream encoder. Data written to +// the returned writer will be encoded and then written to w. +// Ascii85 encodings operate in 32-bit blocks; when finished +// writing, the caller must Close the returned encoder to flush any +// trailing partial block. +func NewEncoder(w io.Writer) io.WriteCloser { return &encoder{w: w} } + +type encoder struct { + err os.Error + w io.Writer + buf [4]byte // buffered data waiting to be encoded + nbuf int // number of bytes in buf + out [1024]byte // output buffer +} + +func (e *encoder) Write(p []byte) (n int, err os.Error) { + if e.err != nil { + return 0, e.err + } + + // Leading fringe. + if e.nbuf > 0 { + var i int + for i = 0; i < len(p) && e.nbuf < 4; i++ { + e.buf[e.nbuf] = p[i] + e.nbuf++ + } + n += i + p = p[i:] + if e.nbuf < 4 { + return + } + nout := Encode(e.out[0:], e.buf[0:]) + if _, e.err = e.w.Write(e.out[0:nout]); e.err != nil { + return n, e.err + } + e.nbuf = 0 + } + + // Large interior chunks. + for len(p) >= 4 { + nn := len(e.out) / 5 * 4 + if nn > len(p) { + nn = len(p) + } + nn -= nn % 4 + if nn > 0 { + nout := Encode(e.out[0:], p[0:nn]) + if _, e.err = e.w.Write(e.out[0:nout]); e.err != nil { + return n, e.err + } + } + n += nn + p = p[nn:] + } + + // Trailing fringe. + for i := 0; i < len(p); i++ { + e.buf[i] = p[i] + } + e.nbuf = len(p) + n += len(p) + return +} + +// Close flushes any pending output from the encoder. +// It is an error to call Write after calling Close. +func (e *encoder) Close() os.Error { + // If there's anything left in the buffer, flush it out + if e.err == nil && e.nbuf > 0 { + nout := Encode(e.out[0:], e.buf[0:e.nbuf]) + e.nbuf = 0 + _, e.err = e.w.Write(e.out[0:nout]) + } + return e.err +} + +/* + * Decoder + */ + +type CorruptInputError int64 + +func (e CorruptInputError) String() string { + return "illegal ascii85 data at input byte " + strconv.Itoa64(int64(e)) +} + +// Decode decodes src into dst, returning both the number +// of bytes written to dst and the number consumed from src. +// If src contains invalid ascii85 data, Decode will return the +// number of bytes successfully written and a CorruptInputError. +// Decode ignores space and control characters in src. +// Often, ascii85-encoded data is wrapped in <~ and ~> symbols. +// Decode expects these to have been stripped by the caller. +// +// If flush is true, Decode assumes that src represents the +// end of the input stream and processes it completely rather +// than wait for the completion of another 32-bit block. +// +// NewDecoder wraps an io.Reader interface around Decode. +// +func Decode(dst, src []byte, flush bool) (ndst, nsrc int, err os.Error) { + var v uint32 + var nb int + for i, b := range src { + if len(dst)-ndst < 4 { + return + } + switch { + case b <= ' ': + continue + case b == 'z' && nb == 0: + nb = 5 + v = 0 + case '!' <= b && b <= 'u': + v = v*85 + uint32(b-'!') + nb++ + default: + return 0, 0, CorruptInputError(i) + } + if nb == 5 { + nsrc = i + 1 + dst[ndst] = byte(v >> 24) + dst[ndst+1] = byte(v >> 16) + dst[ndst+2] = byte(v >> 8) + dst[ndst+3] = byte(v) + ndst += 4 + nb = 0 + v = 0 + } + } + if flush { + nsrc = len(src) + if nb > 0 { + // The number of output bytes in the last fragment + // is the number of leftover input bytes - 1: + // the extra byte provides enough bits to cover + // the inefficiency of the encoding for the block. + if nb == 1 { + return 0, 0, CorruptInputError(len(src)) + } + for i := nb; i < 5; i++ { + // The short encoding truncated the output value. + // We have to assume the worst case values (digit 84) + // in order to ensure that the top bits are correct. + v = v*85 + 84 + } + for i := 0; i < nb-1; i++ { + dst[ndst] = byte(v >> 24) + v <<= 8 + ndst++ + } + } + } + return +} + +// NewDecoder constructs a new ascii85 stream decoder. +func NewDecoder(r io.Reader) io.Reader { return &decoder{r: r} } + +type decoder struct { + err os.Error + readErr os.Error + r io.Reader + end bool // saw end of message + buf [1024]byte // leftover input + nbuf int + out []byte // leftover decoded output + outbuf [1024]byte +} + +func (d *decoder) Read(p []byte) (n int, err os.Error) { + if len(p) == 0 { + return 0, nil + } + if d.err != nil { + return 0, d.err + } + + for { + // Copy leftover output from last decode. + if len(d.out) > 0 { + n = copy(p, d.out) + d.out = d.out[n:] + return + } + + // Decode leftover input from last read. + var nn, nsrc, ndst int + if d.nbuf > 0 { + ndst, nsrc, d.err = Decode(d.outbuf[0:], d.buf[0:d.nbuf], d.readErr != nil) + if ndst > 0 { + d.out = d.outbuf[0:ndst] + d.nbuf = copy(d.buf[0:], d.buf[nsrc:d.nbuf]) + continue // copy out and return + } + } + + // Out of input, out of decoded output. Check errors. + if d.err != nil { + return 0, d.err + } + if d.readErr != nil { + d.err = d.readErr + return 0, d.err + } + + // Read more data. + nn, d.readErr = d.r.Read(d.buf[d.nbuf:]) + d.nbuf += nn + } + panic("unreachable") +} diff --git a/libgo/go/encoding/ascii85/ascii85_test.go b/libgo/go/encoding/ascii85/ascii85_test.go new file mode 100644 index 000000000..fdfeb889f --- /dev/null +++ b/libgo/go/encoding/ascii85/ascii85_test.go @@ -0,0 +1,188 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ascii85 + +import ( + "bytes" + "io/ioutil" + "os" + "testing" +) + +type testpair struct { + decoded, encoded string +} + +var pairs = []testpair{ + // Wikipedia example + { + "Man is distinguished, not only by his reason, but by this singular passion from " + + "other animals, which is a lust of the mind, that by a perseverance of delight in " + + "the continued and indefatigable generation of knowledge, exceeds the short " + + "vehemence of any carnal pleasure.", + "9jqo^BlbD-BleB1DJ+*+F(f,q/0JhKFCj@.4Gp$d7F!,L7@<6@)/0JDEF@3BB/F*&OCAfu2/AKY\n" + + "i(DIb:@FD,*)+C]U=@3BN#EcYf8ATD3s@q?d$AftVqCh[NqF-FD5W8ARlolDIa\n" + + "l(DIduD.RTpAKYo'+CT/5+Cei#DII?(E,9)oF*2M7/c\n", + }, +} + +var bigtest = pairs[len(pairs)-1] + +func testEqual(t *testing.T, msg string, args ...interface{}) bool { + if args[len(args)-2] != args[len(args)-1] { + t.Errorf(msg, args...) + return false + } + return true +} + +func strip85(s string) string { + t := make([]byte, len(s)) + w := 0 + for r := 0; r < len(s); r++ { + c := s[r] + if c > ' ' { + t[w] = c + w++ + } + } + return string(t[0:w]) +} + +func TestEncode(t *testing.T) { + for _, p := range pairs { + buf := make([]byte, MaxEncodedLen(len(p.decoded))) + n := Encode(buf, []byte(p.decoded)) + buf = buf[0:n] + testEqual(t, "Encode(%q) = %q, want %q", p.decoded, strip85(string(buf)), strip85(p.encoded)) + } +} + +func TestEncoder(t *testing.T) { + for _, p := range pairs { + bb := &bytes.Buffer{} + encoder := NewEncoder(bb) + encoder.Write([]byte(p.decoded)) + encoder.Close() + testEqual(t, "Encode(%q) = %q, want %q", p.decoded, strip85(bb.String()), strip85(p.encoded)) + } +} + +func TestEncoderBuffering(t *testing.T) { + input := []byte(bigtest.decoded) + for bs := 1; bs <= 12; bs++ { + bb := &bytes.Buffer{} + encoder := NewEncoder(bb) + for pos := 0; pos < len(input); pos += bs { + end := pos + bs + if end > len(input) { + end = len(input) + } + n, err := encoder.Write(input[pos:end]) + testEqual(t, "Write(%q) gave error %v, want %v", input[pos:end], err, os.Error(nil)) + testEqual(t, "Write(%q) gave length %v, want %v", input[pos:end], n, end-pos) + } + err := encoder.Close() + testEqual(t, "Close gave error %v, want %v", err, os.Error(nil)) + testEqual(t, "Encoding/%d of %q = %q, want %q", bs, bigtest.decoded, strip85(bb.String()), strip85(bigtest.encoded)) + } +} + +func TestDecode(t *testing.T) { + for _, p := range pairs { + dbuf := make([]byte, 4*len(p.encoded)) + ndst, nsrc, err := Decode(dbuf, []byte(p.encoded), true) + testEqual(t, "Decode(%q) = error %v, want %v", p.encoded, err, os.Error(nil)) + testEqual(t, "Decode(%q) = nsrc %v, want %v", p.encoded, nsrc, len(p.encoded)) + testEqual(t, "Decode(%q) = ndst %v, want %v", p.encoded, ndst, len(p.decoded)) + testEqual(t, "Decode(%q) = %q, want %q", p.encoded, string(dbuf[0:ndst]), p.decoded) + } +} + +func TestDecoder(t *testing.T) { + for _, p := range pairs { + decoder := NewDecoder(bytes.NewBufferString(p.encoded)) + dbuf, err := ioutil.ReadAll(decoder) + if err != nil { + t.Fatal("Read failed", err) + } + testEqual(t, "Read from %q = length %v, want %v", p.encoded, len(dbuf), len(p.decoded)) + testEqual(t, "Decoding of %q = %q, want %q", p.encoded, string(dbuf), p.decoded) + if err != nil { + testEqual(t, "Read from %q = %v, want %v", p.encoded, err, os.EOF) + } + } +} + +func TestDecoderBuffering(t *testing.T) { + for bs := 1; bs <= 12; bs++ { + decoder := NewDecoder(bytes.NewBufferString(bigtest.encoded)) + buf := make([]byte, len(bigtest.decoded)+12) + var total int + for total = 0; total < len(bigtest.decoded); { + n, err := decoder.Read(buf[total : total+bs]) + testEqual(t, "Read from %q at pos %d = %d, %v, want _, %v", bigtest.encoded, total, n, err, os.Error(nil)) + total += n + } + testEqual(t, "Decoding/%d of %q = %q, want %q", bs, bigtest.encoded, string(buf[0:total]), bigtest.decoded) + } +} + +func TestDecodeCorrupt(t *testing.T) { + type corrupt struct { + e string + p int + } + examples := []corrupt{ + {"v", 0}, + {"!z!!!!!!!!!", 1}, + } + + for _, e := range examples { + dbuf := make([]byte, 4*len(e.e)) + _, _, err := Decode(dbuf, []byte(e.e), true) + switch err := err.(type) { + case CorruptInputError: + testEqual(t, "Corruption in %q at offset %v, want %v", e.e, int(err), e.p) + default: + t.Error("Decoder failed to detect corruption in", e) + } + } +} + +func TestBig(t *testing.T) { + n := 3*1000 + 1 + raw := make([]byte, n) + const alpha = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + for i := 0; i < n; i++ { + raw[i] = alpha[i%len(alpha)] + } + encoded := new(bytes.Buffer) + w := NewEncoder(encoded) + nn, err := w.Write(raw) + if nn != n || err != nil { + t.Fatalf("Encoder.Write(raw) = %d, %v want %d, nil", nn, err, n) + } + err = w.Close() + if err != nil { + t.Fatalf("Encoder.Close() = %v want nil", err) + } + decoded, err := ioutil.ReadAll(NewDecoder(encoded)) + if err != nil { + t.Fatalf("io.ReadAll(NewDecoder(...)): %v", err) + } + + if !bytes.Equal(raw, decoded) { + var i int + for i = 0; i < len(decoded) && i < len(raw); i++ { + if decoded[i] != raw[i] { + break + } + } + t.Errorf("Decode(Encode(%d-byte string)) failed at offset %d", n, i) + } +} diff --git a/libgo/go/encoding/base32/base32.go b/libgo/go/encoding/base32/base32.go new file mode 100644 index 000000000..acace30d6 --- /dev/null +++ b/libgo/go/encoding/base32/base32.go @@ -0,0 +1,368 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package base32 implements base32 encoding as specified by RFC 4648. +package base32 + +import ( + "io" + "os" + "strconv" +) + +/* + * Encodings + */ + +// An Encoding is a radix 32 encoding/decoding scheme, defined by a +// 32-character alphabet. The most common is the "base32" encoding +// introduced for SASL GSSAPI and standardized in RFC 4648. +// The alternate "base32hex" encoding is used in DNSSEC. +type Encoding struct { + encode string + decodeMap [256]byte +} + +const encodeStd = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567" +const encodeHex = "0123456789ABCDEFGHIJKLMNOPQRSTUV" + +// NewEncoding returns a new Encoding defined by the given alphabet, +// which must be a 32-byte string. +func NewEncoding(encoder string) *Encoding { + e := new(Encoding) + e.encode = encoder + for i := 0; i < len(e.decodeMap); i++ { + e.decodeMap[i] = 0xFF + } + for i := 0; i < len(encoder); i++ { + e.decodeMap[encoder[i]] = byte(i) + } + return e +} + +// StdEncoding is the standard base32 encoding, as defined in +// RFC 4648. +var StdEncoding = NewEncoding(encodeStd) + +// HexEncoding is the ``Extended Hex Alphabet'' defined in RFC 4648. +// It is typically used in DNS. +var HexEncoding = NewEncoding(encodeHex) + +/* + * Encoder + */ + +// Encode encodes src using the encoding enc, writing +// EncodedLen(len(src)) bytes to dst. +// +// The encoding pads the output to a multiple of 8 bytes, +// so Encode is not appropriate for use on individual blocks +// of a large data stream. Use NewEncoder() instead. +func (enc *Encoding) Encode(dst, src []byte) { + if len(src) == 0 { + return + } + + for len(src) > 0 { + dst[0] = 0 + dst[1] = 0 + dst[2] = 0 + dst[3] = 0 + dst[4] = 0 + dst[5] = 0 + dst[6] = 0 + dst[7] = 0 + + // Unpack 8x 5-bit source blocks into a 5 byte + // destination quantum + switch len(src) { + default: + dst[7] |= src[4] & 0x1F + dst[6] |= src[4] >> 5 + fallthrough + case 4: + dst[6] |= (src[3] << 3) & 0x1F + dst[5] |= (src[3] >> 2) & 0x1F + dst[4] |= src[3] >> 7 + fallthrough + case 3: + dst[4] |= (src[2] << 1) & 0x1F + dst[3] |= (src[2] >> 4) & 0x1F + fallthrough + case 2: + dst[3] |= (src[1] << 4) & 0x1F + dst[2] |= (src[1] >> 1) & 0x1F + dst[1] |= (src[1] >> 6) & 0x1F + fallthrough + case 1: + dst[1] |= (src[0] << 2) & 0x1F + dst[0] |= src[0] >> 3 + } + + // Encode 5-bit blocks using the base32 alphabet + for j := 0; j < 8; j++ { + dst[j] = enc.encode[dst[j]] + } + + // Pad the final quantum + if len(src) < 5 { + dst[7] = '=' + if len(src) < 4 { + dst[6] = '=' + dst[5] = '=' + if len(src) < 3 { + dst[4] = '=' + if len(src) < 2 { + dst[3] = '=' + dst[2] = '=' + } + } + } + break + } + src = src[5:] + dst = dst[8:] + } +} + +type encoder struct { + err os.Error + enc *Encoding + w io.Writer + buf [5]byte // buffered data waiting to be encoded + nbuf int // number of bytes in buf + out [1024]byte // output buffer +} + +func (e *encoder) Write(p []byte) (n int, err os.Error) { + if e.err != nil { + return 0, e.err + } + + // Leading fringe. + if e.nbuf > 0 { + var i int + for i = 0; i < len(p) && e.nbuf < 5; i++ { + e.buf[e.nbuf] = p[i] + e.nbuf++ + } + n += i + p = p[i:] + if e.nbuf < 5 { + return + } + e.enc.Encode(e.out[0:], e.buf[0:]) + if _, e.err = e.w.Write(e.out[0:8]); e.err != nil { + return n, e.err + } + e.nbuf = 0 + } + + // Large interior chunks. + for len(p) >= 5 { + nn := len(e.out) / 8 * 5 + if nn > len(p) { + nn = len(p) + } + nn -= nn % 5 + if nn > 0 { + e.enc.Encode(e.out[0:], p[0:nn]) + if _, e.err = e.w.Write(e.out[0 : nn/5*8]); e.err != nil { + return n, e.err + } + } + n += nn + p = p[nn:] + } + + // Trailing fringe. + for i := 0; i < len(p); i++ { + e.buf[i] = p[i] + } + e.nbuf = len(p) + n += len(p) + return +} + +// Close flushes any pending output from the encoder. +// It is an error to call Write after calling Close. +func (e *encoder) Close() os.Error { + // If there's anything left in the buffer, flush it out + if e.err == nil && e.nbuf > 0 { + e.enc.Encode(e.out[0:], e.buf[0:e.nbuf]) + e.nbuf = 0 + _, e.err = e.w.Write(e.out[0:8]) + } + return e.err +} + +// NewEncoder returns a new base32 stream encoder. Data written to +// the returned writer will be encoded using enc and then written to w. +// Base32 encodings operate in 5-byte blocks; when finished +// writing, the caller must Close the returned encoder to flush any +// partially written blocks. +func NewEncoder(enc *Encoding, w io.Writer) io.WriteCloser { + return &encoder{enc: enc, w: w} +} + +// EncodedLen returns the length in bytes of the base32 encoding +// of an input buffer of length n. +func (enc *Encoding) EncodedLen(n int) int { return (n + 4) / 5 * 8 } + +/* + * Decoder + */ + +type CorruptInputError int64 + +func (e CorruptInputError) String() string { + return "illegal base32 data at input byte " + strconv.Itoa64(int64(e)) +} + +// decode is like Decode but returns an additional 'end' value, which +// indicates if end-of-message padding was encountered and thus any +// additional data is an error. decode also assumes len(src)%8==0, +// since it is meant for internal use. +func (enc *Encoding) decode(dst, src []byte) (n int, end bool, err os.Error) { + for i := 0; i < len(src)/8 && !end; i++ { + // Decode quantum using the base32 alphabet + var dbuf [8]byte + dlen := 8 + + // do the top bytes contain any data? + dbufloop: + for j := 0; j < 8; j++ { + in := src[i*8+j] + if in == '=' && j >= 2 && i == len(src)/8-1 { + // We've reached the end and there's + // padding, the rest should be padded + for k := j; k < 8; k++ { + if src[i*8+k] != '=' { + return n, false, CorruptInputError(i*8 + j) + } + } + dlen = j + end = true + break dbufloop + } + dbuf[j] = enc.decodeMap[in] + if dbuf[j] == 0xFF { + return n, false, CorruptInputError(i*8 + j) + } + } + + // Pack 8x 5-bit source blocks into 5 byte destination + // quantum + switch dlen { + case 7, 8: + dst[i*5+4] = dbuf[6]<<5 | dbuf[7] + fallthrough + case 6, 5: + dst[i*5+3] = dbuf[4]<<7 | dbuf[5]<<2 | dbuf[6]>>3 + fallthrough + case 4: + dst[i*5+2] = dbuf[3]<<4 | dbuf[4]>>1 + fallthrough + case 3: + dst[i*5+1] = dbuf[1]<<6 | dbuf[2]<<1 | dbuf[3]>>4 + fallthrough + case 2: + dst[i*5+0] = dbuf[0]<<3 | dbuf[1]>>2 + } + switch dlen { + case 2: + n += 1 + case 3, 4: + n += 2 + case 5: + n += 3 + case 6, 7: + n += 4 + case 8: + n += 5 + } + } + return n, end, nil +} + +// Decode decodes src using the encoding enc. It writes at most +// DecodedLen(len(src)) bytes to dst and returns the number of bytes +// written. If src contains invalid base32 data, it will return the +// number of bytes successfully written and CorruptInputError. +func (enc *Encoding) Decode(dst, src []byte) (n int, err os.Error) { + if len(src)%8 != 0 { + return 0, CorruptInputError(len(src) / 8 * 8) + } + + n, _, err = enc.decode(dst, src) + return +} + +type decoder struct { + err os.Error + enc *Encoding + r io.Reader + end bool // saw end of message + buf [1024]byte // leftover input + nbuf int + out []byte // leftover decoded output + outbuf [1024 / 8 * 5]byte +} + +func (d *decoder) Read(p []byte) (n int, err os.Error) { + if d.err != nil { + return 0, d.err + } + + // Use leftover decoded output from last read. + if len(d.out) > 0 { + n = copy(p, d.out) + d.out = d.out[n:] + return n, nil + } + + // Read a chunk. + nn := len(p) / 5 * 8 + if nn < 8 { + nn = 8 + } + if nn > len(d.buf) { + nn = len(d.buf) + } + nn, d.err = io.ReadAtLeast(d.r, d.buf[d.nbuf:nn], 8-d.nbuf) + d.nbuf += nn + if d.nbuf < 8 { + return 0, d.err + } + + // Decode chunk into p, or d.out and then p if p is too small. + nr := d.nbuf / 8 * 8 + nw := d.nbuf / 8 * 5 + if nw > len(p) { + nw, d.end, d.err = d.enc.decode(d.outbuf[0:], d.buf[0:nr]) + d.out = d.outbuf[0:nw] + n = copy(p, d.out) + d.out = d.out[n:] + } else { + n, d.end, d.err = d.enc.decode(p, d.buf[0:nr]) + } + d.nbuf -= nr + for i := 0; i < d.nbuf; i++ { + d.buf[i] = d.buf[i+nr] + } + + if d.err == nil { + d.err = err + } + return n, d.err +} + +// NewDecoder constructs a new base32 stream decoder. +func NewDecoder(enc *Encoding, r io.Reader) io.Reader { + return &decoder{enc: enc, r: r} +} + +// DecodedLen returns the maximum length in bytes of the decoded data +// corresponding to n bytes of base32-encoded data. +func (enc *Encoding) DecodedLen(n int) int { return n / 8 * 5 } diff --git a/libgo/go/encoding/base32/base32_test.go b/libgo/go/encoding/base32/base32_test.go new file mode 100644 index 000000000..792e4dc63 --- /dev/null +++ b/libgo/go/encoding/base32/base32_test.go @@ -0,0 +1,194 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package base32 + +import ( + "bytes" + "io/ioutil" + "os" + "testing" +) + +type testpair struct { + decoded, encoded string +} + +var pairs = []testpair{ + // RFC 4648 examples + {"", ""}, + {"f", "MY======"}, + {"fo", "MZXQ===="}, + {"foo", "MZXW6==="}, + {"foob", "MZXW6YQ="}, + {"fooba", "MZXW6YTB"}, + {"foobar", "MZXW6YTBOI======"}, + + + // Wikipedia examples, converted to base32 + {"sure.", "ON2XEZJO"}, + {"sure", "ON2XEZI="}, + {"sur", "ON2XE==="}, + {"su", "ON2Q===="}, + {"leasure.", "NRSWC43VOJSS4==="}, + {"easure.", "MVQXG5LSMUXA===="}, + {"asure.", "MFZXK4TFFY======"}, + {"sure.", "ON2XEZJO"}, +} + +var bigtest = testpair{ + "Twas brillig, and the slithy toves", + "KR3WC4ZAMJZGS3DMNFTSYIDBNZSCA5DIMUQHG3DJORUHSIDUN53GK4Y=", +} + +func testEqual(t *testing.T, msg string, args ...interface{}) bool { + if args[len(args)-2] != args[len(args)-1] { + t.Errorf(msg, args...) + return false + } + return true +} + +func TestEncode(t *testing.T) { + for _, p := range pairs { + buf := make([]byte, StdEncoding.EncodedLen(len(p.decoded))) + StdEncoding.Encode(buf, []byte(p.decoded)) + testEqual(t, "Encode(%q) = %q, want %q", p.decoded, string(buf), p.encoded) + } +} + +func TestEncoder(t *testing.T) { + for _, p := range pairs { + bb := &bytes.Buffer{} + encoder := NewEncoder(StdEncoding, bb) + encoder.Write([]byte(p.decoded)) + encoder.Close() + testEqual(t, "Encode(%q) = %q, want %q", p.decoded, bb.String(), p.encoded) + } +} + +func TestEncoderBuffering(t *testing.T) { + input := []byte(bigtest.decoded) + for bs := 1; bs <= 12; bs++ { + bb := &bytes.Buffer{} + encoder := NewEncoder(StdEncoding, bb) + for pos := 0; pos < len(input); pos += bs { + end := pos + bs + if end > len(input) { + end = len(input) + } + n, err := encoder.Write(input[pos:end]) + testEqual(t, "Write(%q) gave error %v, want %v", input[pos:end], err, os.Error(nil)) + testEqual(t, "Write(%q) gave length %v, want %v", input[pos:end], n, end-pos) + } + err := encoder.Close() + testEqual(t, "Close gave error %v, want %v", err, os.Error(nil)) + testEqual(t, "Encoding/%d of %q = %q, want %q", bs, bigtest.decoded, bb.String(), bigtest.encoded) + } +} + +func TestDecode(t *testing.T) { + for _, p := range pairs { + dbuf := make([]byte, StdEncoding.DecodedLen(len(p.encoded))) + count, end, err := StdEncoding.decode(dbuf, []byte(p.encoded)) + testEqual(t, "Decode(%q) = error %v, want %v", p.encoded, err, os.Error(nil)) + testEqual(t, "Decode(%q) = length %v, want %v", p.encoded, count, len(p.decoded)) + if len(p.encoded) > 0 { + testEqual(t, "Decode(%q) = end %v, want %v", p.encoded, end, (p.encoded[len(p.encoded)-1] == '=')) + } + testEqual(t, "Decode(%q) = %q, want %q", p.encoded, + string(dbuf[0:count]), + p.decoded) + } +} + +func TestDecoder(t *testing.T) { + for _, p := range pairs { + decoder := NewDecoder(StdEncoding, bytes.NewBufferString(p.encoded)) + dbuf := make([]byte, StdEncoding.DecodedLen(len(p.encoded))) + count, err := decoder.Read(dbuf) + if err != nil && err != os.EOF { + t.Fatal("Read failed", err) + } + testEqual(t, "Read from %q = length %v, want %v", p.encoded, count, len(p.decoded)) + testEqual(t, "Decoding of %q = %q, want %q", p.encoded, string(dbuf[0:count]), p.decoded) + if err != os.EOF { + count, err = decoder.Read(dbuf) + } + testEqual(t, "Read from %q = %v, want %v", p.encoded, err, os.EOF) + } +} + +func TestDecoderBuffering(t *testing.T) { + for bs := 1; bs <= 12; bs++ { + decoder := NewDecoder(StdEncoding, bytes.NewBufferString(bigtest.encoded)) + buf := make([]byte, len(bigtest.decoded)+12) + var total int + for total = 0; total < len(bigtest.decoded); { + n, err := decoder.Read(buf[total : total+bs]) + testEqual(t, "Read from %q at pos %d = %d, %v, want _, %v", bigtest.encoded, total, n, err, os.Error(nil)) + total += n + } + testEqual(t, "Decoding/%d of %q = %q, want %q", bs, bigtest.encoded, string(buf[0:total]), bigtest.decoded) + } +} + +func TestDecodeCorrupt(t *testing.T) { + type corrupt struct { + e string + p int + } + examples := []corrupt{ + {"!!!!", 0}, + {"x===", 0}, + {"AA=A====", 2}, + {"AAA=AAAA", 3}, + {"MMMMMMMMM", 8}, + {"MMMMMM", 0}, + } + + for _, e := range examples { + dbuf := make([]byte, StdEncoding.DecodedLen(len(e.e))) + _, err := StdEncoding.Decode(dbuf, []byte(e.e)) + switch err := err.(type) { + case CorruptInputError: + testEqual(t, "Corruption in %q at offset %v, want %v", e.e, int(err), e.p) + default: + t.Error("Decoder failed to detect corruption in", e) + } + } +} + +func TestBig(t *testing.T) { + n := 3*1000 + 1 + raw := make([]byte, n) + const alpha = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + for i := 0; i < n; i++ { + raw[i] = alpha[i%len(alpha)] + } + encoded := new(bytes.Buffer) + w := NewEncoder(StdEncoding, encoded) + nn, err := w.Write(raw) + if nn != n || err != nil { + t.Fatalf("Encoder.Write(raw) = %d, %v want %d, nil", nn, err, n) + } + err = w.Close() + if err != nil { + t.Fatalf("Encoder.Close() = %v want nil", err) + } + decoded, err := ioutil.ReadAll(NewDecoder(StdEncoding, encoded)) + if err != nil { + t.Fatalf("ioutil.ReadAll(NewDecoder(...)): %v", err) + } + + if !bytes.Equal(raw, decoded) { + var i int + for i = 0; i < len(decoded) && i < len(raw); i++ { + if decoded[i] != raw[i] { + break + } + } + t.Errorf("Decode(Encode(%d-byte string)) failed at offset %d", n, i) + } +} diff --git a/libgo/go/encoding/base64/base64.go b/libgo/go/encoding/base64/base64.go new file mode 100644 index 000000000..496129798 --- /dev/null +++ b/libgo/go/encoding/base64/base64.go @@ -0,0 +1,329 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package base64 implements base64 encoding as specified by RFC 4648. +package base64 + +import ( + "io" + "os" + "strconv" +) + +/* + * Encodings + */ + +// An Encoding is a radix 64 encoding/decoding scheme, defined by a +// 64-character alphabet. The most common encoding is the "base64" +// encoding defined in RFC 4648 and used in MIME (RFC 2045) and PEM +// (RFC 1421). RFC 4648 also defines an alternate encoding, which is +// the standard encoding with - and _ substituted for + and /. +type Encoding struct { + encode string + decodeMap [256]byte +} + +const encodeStd = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" +const encodeURL = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_" + +// NewEncoding returns a new Encoding defined by the given alphabet, +// which must be a 64-byte string. +func NewEncoding(encoder string) *Encoding { + e := new(Encoding) + e.encode = encoder + for i := 0; i < len(e.decodeMap); i++ { + e.decodeMap[i] = 0xFF + } + for i := 0; i < len(encoder); i++ { + e.decodeMap[encoder[i]] = byte(i) + } + return e +} + +// StdEncoding is the standard base64 encoding, as defined in +// RFC 4648. +var StdEncoding = NewEncoding(encodeStd) + +// URLEncoding is the alternate base64 encoding defined in RFC 4648. +// It is typically used in URLs and file names. +var URLEncoding = NewEncoding(encodeURL) + +/* + * Encoder + */ + +// Encode encodes src using the encoding enc, writing +// EncodedLen(len(src)) bytes to dst. +// +// The encoding pads the output to a multiple of 4 bytes, +// so Encode is not appropriate for use on individual blocks +// of a large data stream. Use NewEncoder() instead. +func (enc *Encoding) Encode(dst, src []byte) { + if len(src) == 0 { + return + } + + for len(src) > 0 { + dst[0] = 0 + dst[1] = 0 + dst[2] = 0 + dst[3] = 0 + + // Unpack 4x 6-bit source blocks into a 4 byte + // destination quantum + switch len(src) { + default: + dst[3] |= src[2] & 0x3F + dst[2] |= src[2] >> 6 + fallthrough + case 2: + dst[2] |= (src[1] << 2) & 0x3F + dst[1] |= src[1] >> 4 + fallthrough + case 1: + dst[1] |= (src[0] << 4) & 0x3F + dst[0] |= src[0] >> 2 + } + + // Encode 6-bit blocks using the base64 alphabet + for j := 0; j < 4; j++ { + dst[j] = enc.encode[dst[j]] + } + + // Pad the final quantum + if len(src) < 3 { + dst[3] = '=' + if len(src) < 2 { + dst[2] = '=' + } + break + } + + src = src[3:] + dst = dst[4:] + } +} + +type encoder struct { + err os.Error + enc *Encoding + w io.Writer + buf [3]byte // buffered data waiting to be encoded + nbuf int // number of bytes in buf + out [1024]byte // output buffer +} + +func (e *encoder) Write(p []byte) (n int, err os.Error) { + if e.err != nil { + return 0, e.err + } + + // Leading fringe. + if e.nbuf > 0 { + var i int + for i = 0; i < len(p) && e.nbuf < 3; i++ { + e.buf[e.nbuf] = p[i] + e.nbuf++ + } + n += i + p = p[i:] + if e.nbuf < 3 { + return + } + e.enc.Encode(e.out[0:], e.buf[0:]) + if _, e.err = e.w.Write(e.out[0:4]); e.err != nil { + return n, e.err + } + e.nbuf = 0 + } + + // Large interior chunks. + for len(p) >= 3 { + nn := len(e.out) / 4 * 3 + if nn > len(p) { + nn = len(p) + } + nn -= nn % 3 + if nn > 0 { + e.enc.Encode(e.out[0:], p[0:nn]) + if _, e.err = e.w.Write(e.out[0 : nn/3*4]); e.err != nil { + return n, e.err + } + } + n += nn + p = p[nn:] + } + + // Trailing fringe. + for i := 0; i < len(p); i++ { + e.buf[i] = p[i] + } + e.nbuf = len(p) + n += len(p) + return +} + +// Close flushes any pending output from the encoder. +// It is an error to call Write after calling Close. +func (e *encoder) Close() os.Error { + // If there's anything left in the buffer, flush it out + if e.err == nil && e.nbuf > 0 { + e.enc.Encode(e.out[0:], e.buf[0:e.nbuf]) + e.nbuf = 0 + _, e.err = e.w.Write(e.out[0:4]) + } + return e.err +} + +// NewEncoder returns a new base64 stream encoder. Data written to +// the returned writer will be encoded using enc and then written to w. +// Base64 encodings operate in 4-byte blocks; when finished +// writing, the caller must Close the returned encoder to flush any +// partially written blocks. +func NewEncoder(enc *Encoding, w io.Writer) io.WriteCloser { + return &encoder{enc: enc, w: w} +} + +// EncodedLen returns the length in bytes of the base64 encoding +// of an input buffer of length n. +func (enc *Encoding) EncodedLen(n int) int { return (n + 2) / 3 * 4 } + +/* + * Decoder + */ + +type CorruptInputError int64 + +func (e CorruptInputError) String() string { + return "illegal base64 data at input byte " + strconv.Itoa64(int64(e)) +} + +// decode is like Decode but returns an additional 'end' value, which +// indicates if end-of-message padding was encountered and thus any +// additional data is an error. decode also assumes len(src)%4==0, +// since it is meant for internal use. +func (enc *Encoding) decode(dst, src []byte) (n int, end bool, err os.Error) { + for i := 0; i < len(src)/4 && !end; i++ { + // Decode quantum using the base64 alphabet + var dbuf [4]byte + dlen := 4 + + dbufloop: + for j := 0; j < 4; j++ { + in := src[i*4+j] + if in == '=' && j >= 2 && i == len(src)/4-1 { + // We've reached the end and there's + // padding + if src[i*4+3] != '=' { + return n, false, CorruptInputError(i*4 + 2) + } + dlen = j + end = true + break dbufloop + } + dbuf[j] = enc.decodeMap[in] + if dbuf[j] == 0xFF { + return n, false, CorruptInputError(i*4 + j) + } + } + + // Pack 4x 6-bit source blocks into 3 byte destination + // quantum + switch dlen { + case 4: + dst[i*3+2] = dbuf[2]<<6 | dbuf[3] + fallthrough + case 3: + dst[i*3+1] = dbuf[1]<<4 | dbuf[2]>>2 + fallthrough + case 2: + dst[i*3+0] = dbuf[0]<<2 | dbuf[1]>>4 + } + n += dlen - 1 + } + + return n, end, nil +} + +// Decode decodes src using the encoding enc. It writes at most +// DecodedLen(len(src)) bytes to dst and returns the number of bytes +// written. If src contains invalid base64 data, it will return the +// number of bytes successfully written and CorruptInputError. +func (enc *Encoding) Decode(dst, src []byte) (n int, err os.Error) { + if len(src)%4 != 0 { + return 0, CorruptInputError(len(src) / 4 * 4) + } + + n, _, err = enc.decode(dst, src) + return +} + +type decoder struct { + err os.Error + enc *Encoding + r io.Reader + end bool // saw end of message + buf [1024]byte // leftover input + nbuf int + out []byte // leftover decoded output + outbuf [1024 / 4 * 3]byte +} + +func (d *decoder) Read(p []byte) (n int, err os.Error) { + if d.err != nil { + return 0, d.err + } + + // Use leftover decoded output from last read. + if len(d.out) > 0 { + n = copy(p, d.out) + d.out = d.out[n:] + return n, nil + } + + // Read a chunk. + nn := len(p) / 3 * 4 + if nn < 4 { + nn = 4 + } + if nn > len(d.buf) { + nn = len(d.buf) + } + nn, d.err = io.ReadAtLeast(d.r, d.buf[d.nbuf:nn], 4-d.nbuf) + d.nbuf += nn + if d.nbuf < 4 { + return 0, d.err + } + + // Decode chunk into p, or d.out and then p if p is too small. + nr := d.nbuf / 4 * 4 + nw := d.nbuf / 4 * 3 + if nw > len(p) { + nw, d.end, d.err = d.enc.decode(d.outbuf[0:], d.buf[0:nr]) + d.out = d.outbuf[0:nw] + n = copy(p, d.out) + d.out = d.out[n:] + } else { + n, d.end, d.err = d.enc.decode(p, d.buf[0:nr]) + } + d.nbuf -= nr + for i := 0; i < d.nbuf; i++ { + d.buf[i] = d.buf[i+nr] + } + + if d.err == nil { + d.err = err + } + return n, d.err +} + +// NewDecoder constructs a new base64 stream decoder. +func NewDecoder(enc *Encoding, r io.Reader) io.Reader { + return &decoder{enc: enc, r: r} +} + +// DecodedLen returns the maximum length in bytes of the decoded data +// corresponding to n bytes of base64-encoded data. +func (enc *Encoding) DecodedLen(n int) int { return n / 4 * 3 } diff --git a/libgo/go/encoding/base64/base64_test.go b/libgo/go/encoding/base64/base64_test.go new file mode 100644 index 000000000..de41e704b --- /dev/null +++ b/libgo/go/encoding/base64/base64_test.go @@ -0,0 +1,196 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package base64 + +import ( + "bytes" + "io/ioutil" + "os" + "testing" +) + +type testpair struct { + decoded, encoded string +} + +var pairs = []testpair{ + // RFC 3548 examples + {"\x14\xfb\x9c\x03\xd9\x7e", "FPucA9l+"}, + {"\x14\xfb\x9c\x03\xd9", "FPucA9k="}, + {"\x14\xfb\x9c\x03", "FPucAw=="}, + + // RFC 4648 examples + {"", ""}, + {"f", "Zg=="}, + {"fo", "Zm8="}, + {"foo", "Zm9v"}, + {"foob", "Zm9vYg=="}, + {"fooba", "Zm9vYmE="}, + {"foobar", "Zm9vYmFy"}, + + // Wikipedia examples + {"sure.", "c3VyZS4="}, + {"sure", "c3VyZQ=="}, + {"sur", "c3Vy"}, + {"su", "c3U="}, + {"leasure.", "bGVhc3VyZS4="}, + {"easure.", "ZWFzdXJlLg=="}, + {"asure.", "YXN1cmUu"}, + {"sure.", "c3VyZS4="}, +} + +var bigtest = testpair{ + "Twas brillig, and the slithy toves", + "VHdhcyBicmlsbGlnLCBhbmQgdGhlIHNsaXRoeSB0b3Zlcw==", +} + +func testEqual(t *testing.T, msg string, args ...interface{}) bool { + if args[len(args)-2] != args[len(args)-1] { + t.Errorf(msg, args...) + return false + } + return true +} + +func TestEncode(t *testing.T) { + for _, p := range pairs { + buf := make([]byte, StdEncoding.EncodedLen(len(p.decoded))) + StdEncoding.Encode(buf, []byte(p.decoded)) + testEqual(t, "Encode(%q) = %q, want %q", p.decoded, string(buf), p.encoded) + } +} + +func TestEncoder(t *testing.T) { + for _, p := range pairs { + bb := &bytes.Buffer{} + encoder := NewEncoder(StdEncoding, bb) + encoder.Write([]byte(p.decoded)) + encoder.Close() + testEqual(t, "Encode(%q) = %q, want %q", p.decoded, bb.String(), p.encoded) + } +} + +func TestEncoderBuffering(t *testing.T) { + input := []byte(bigtest.decoded) + for bs := 1; bs <= 12; bs++ { + bb := &bytes.Buffer{} + encoder := NewEncoder(StdEncoding, bb) + for pos := 0; pos < len(input); pos += bs { + end := pos + bs + if end > len(input) { + end = len(input) + } + n, err := encoder.Write(input[pos:end]) + testEqual(t, "Write(%q) gave error %v, want %v", input[pos:end], err, os.Error(nil)) + testEqual(t, "Write(%q) gave length %v, want %v", input[pos:end], n, end-pos) + } + err := encoder.Close() + testEqual(t, "Close gave error %v, want %v", err, os.Error(nil)) + testEqual(t, "Encoding/%d of %q = %q, want %q", bs, bigtest.decoded, bb.String(), bigtest.encoded) + } +} + +func TestDecode(t *testing.T) { + for _, p := range pairs { + dbuf := make([]byte, StdEncoding.DecodedLen(len(p.encoded))) + count, end, err := StdEncoding.decode(dbuf, []byte(p.encoded)) + testEqual(t, "Decode(%q) = error %v, want %v", p.encoded, err, os.Error(nil)) + testEqual(t, "Decode(%q) = length %v, want %v", p.encoded, count, len(p.decoded)) + if len(p.encoded) > 0 { + testEqual(t, "Decode(%q) = end %v, want %v", p.encoded, end, (p.encoded[len(p.encoded)-1] == '=')) + } + testEqual(t, "Decode(%q) = %q, want %q", p.encoded, string(dbuf[0:count]), p.decoded) + } +} + +func TestDecoder(t *testing.T) { + for _, p := range pairs { + decoder := NewDecoder(StdEncoding, bytes.NewBufferString(p.encoded)) + dbuf := make([]byte, StdEncoding.DecodedLen(len(p.encoded))) + count, err := decoder.Read(dbuf) + if err != nil && err != os.EOF { + t.Fatal("Read failed", err) + } + testEqual(t, "Read from %q = length %v, want %v", p.encoded, count, len(p.decoded)) + testEqual(t, "Decoding of %q = %q, want %q", p.encoded, string(dbuf[0:count]), p.decoded) + if err != os.EOF { + count, err = decoder.Read(dbuf) + } + testEqual(t, "Read from %q = %v, want %v", p.encoded, err, os.EOF) + } +} + +func TestDecoderBuffering(t *testing.T) { + for bs := 1; bs <= 12; bs++ { + decoder := NewDecoder(StdEncoding, bytes.NewBufferString(bigtest.encoded)) + buf := make([]byte, len(bigtest.decoded)+12) + var total int + for total = 0; total < len(bigtest.decoded); { + n, err := decoder.Read(buf[total : total+bs]) + testEqual(t, "Read from %q at pos %d = %d, %v, want _, %v", bigtest.encoded, total, n, err, os.Error(nil)) + total += n + } + testEqual(t, "Decoding/%d of %q = %q, want %q", bs, bigtest.encoded, string(buf[0:total]), bigtest.decoded) + } +} + +func TestDecodeCorrupt(t *testing.T) { + type corrupt struct { + e string + p int + } + examples := []corrupt{ + {"!!!!", 0}, + {"x===", 1}, + {"AA=A", 2}, + {"AAA=AAAA", 3}, + {"AAAAA", 4}, + {"AAAAAA", 4}, + } + + for _, e := range examples { + dbuf := make([]byte, StdEncoding.DecodedLen(len(e.e))) + _, err := StdEncoding.Decode(dbuf, []byte(e.e)) + switch err := err.(type) { + case CorruptInputError: + testEqual(t, "Corruption in %q at offset %v, want %v", e.e, int(err), e.p) + default: + t.Error("Decoder failed to detect corruption in", e) + } + } +} + +func TestBig(t *testing.T) { + n := 3*1000 + 1 + raw := make([]byte, n) + const alpha = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + for i := 0; i < n; i++ { + raw[i] = alpha[i%len(alpha)] + } + encoded := new(bytes.Buffer) + w := NewEncoder(StdEncoding, encoded) + nn, err := w.Write(raw) + if nn != n || err != nil { + t.Fatalf("Encoder.Write(raw) = %d, %v want %d, nil", nn, err, n) + } + err = w.Close() + if err != nil { + t.Fatalf("Encoder.Close() = %v want nil", err) + } + decoded, err := ioutil.ReadAll(NewDecoder(StdEncoding, encoded)) + if err != nil { + t.Fatalf("ioutil.ReadAll(NewDecoder(...)): %v", err) + } + + if !bytes.Equal(raw, decoded) { + var i int + for i = 0; i < len(decoded) && i < len(raw); i++ { + if decoded[i] != raw[i] { + break + } + } + t.Errorf("Decode(Encode(%d-byte string)) failed at offset %d", n, i) + } +} diff --git a/libgo/go/encoding/binary/binary.go b/libgo/go/encoding/binary/binary.go new file mode 100644 index 000000000..77ff3a9f3 --- /dev/null +++ b/libgo/go/encoding/binary/binary.go @@ -0,0 +1,409 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This package implements translation between +// unsigned integer values and byte sequences. +package binary + +import ( + "math" + "io" + "os" + "reflect" +) + +// A ByteOrder specifies how to convert byte sequences into +// 16-, 32-, or 64-bit unsigned integers. +type ByteOrder interface { + Uint16(b []byte) uint16 + Uint32(b []byte) uint32 + Uint64(b []byte) uint64 + PutUint16([]byte, uint16) + PutUint32([]byte, uint32) + PutUint64([]byte, uint64) + String() string +} + +// This is byte instead of struct{} so that it can be compared, +// allowing, e.g., order == binary.LittleEndian. +type unused byte + +// LittleEndian is the little-endian implementation of ByteOrder. +var LittleEndian littleEndian + +// BigEndian is the big-endian implementation of ByteOrder. +var BigEndian bigEndian + +type littleEndian unused + +func (littleEndian) Uint16(b []byte) uint16 { return uint16(b[0]) | uint16(b[1])<<8 } + +func (littleEndian) PutUint16(b []byte, v uint16) { + b[0] = byte(v) + b[1] = byte(v >> 8) +} + +func (littleEndian) Uint32(b []byte) uint32 { + return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24 +} + +func (littleEndian) PutUint32(b []byte, v uint32) { + b[0] = byte(v) + b[1] = byte(v >> 8) + b[2] = byte(v >> 16) + b[3] = byte(v >> 24) +} + +func (littleEndian) Uint64(b []byte) uint64 { + return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | + uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 +} + +func (littleEndian) PutUint64(b []byte, v uint64) { + b[0] = byte(v) + b[1] = byte(v >> 8) + b[2] = byte(v >> 16) + b[3] = byte(v >> 24) + b[4] = byte(v >> 32) + b[5] = byte(v >> 40) + b[6] = byte(v >> 48) + b[7] = byte(v >> 56) +} + +func (littleEndian) String() string { return "LittleEndian" } + +func (littleEndian) GoString() string { return "binary.LittleEndian" } + +type bigEndian unused + +func (bigEndian) Uint16(b []byte) uint16 { return uint16(b[1]) | uint16(b[0])<<8 } + +func (bigEndian) PutUint16(b []byte, v uint16) { + b[0] = byte(v >> 8) + b[1] = byte(v) +} + +func (bigEndian) Uint32(b []byte) uint32 { + return uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24 +} + +func (bigEndian) PutUint32(b []byte, v uint32) { + b[0] = byte(v >> 24) + b[1] = byte(v >> 16) + b[2] = byte(v >> 8) + b[3] = byte(v) +} + +func (bigEndian) Uint64(b []byte) uint64 { + return uint64(b[7]) | uint64(b[6])<<8 | uint64(b[5])<<16 | uint64(b[4])<<24 | + uint64(b[3])<<32 | uint64(b[2])<<40 | uint64(b[1])<<48 | uint64(b[0])<<56 +} + +func (bigEndian) PutUint64(b []byte, v uint64) { + b[0] = byte(v >> 56) + b[1] = byte(v >> 48) + b[2] = byte(v >> 40) + b[3] = byte(v >> 32) + b[4] = byte(v >> 24) + b[5] = byte(v >> 16) + b[6] = byte(v >> 8) + b[7] = byte(v) +} + +func (bigEndian) String() string { return "BigEndian" } + +func (bigEndian) GoString() string { return "binary.BigEndian" } + +// Read reads structured binary data from r into data. +// Data must be a pointer to a fixed-size value or a slice +// of fixed-size values. +// A fixed-size value is either a fixed-size arithmetic +// type (int8, uint8, int16, float32, complex64, ...) +// or an array or struct containing only fixed-size values. +// Bytes read from r are decoded using the specified byte order +// and written to successive fields of the data. +func Read(r io.Reader, order ByteOrder, data interface{}) os.Error { + var v reflect.Value + switch d := reflect.NewValue(data).(type) { + case *reflect.PtrValue: + v = d.Elem() + case *reflect.SliceValue: + v = d + default: + return os.NewError("binary.Read: invalid type " + d.Type().String()) + } + size := TotalSize(v) + if size < 0 { + return os.NewError("binary.Read: invalid type " + v.Type().String()) + } + d := &decoder{order: order, buf: make([]byte, size)} + if _, err := io.ReadFull(r, d.buf); err != nil { + return err + } + d.value(v) + return nil +} + +// Write writes the binary representation of data into w. +// Data must be a fixed-size value or a pointer to +// a fixed-size value. +// A fixed-size value is either a fixed-size arithmetic +// type (int8, uint8, int16, float32, complex64, ...) +// or an array or struct containing only fixed-size values. +// Bytes written to w are encoded using the specified byte order +// and read from successive fields of the data. +func Write(w io.Writer, order ByteOrder, data interface{}) os.Error { + v := reflect.Indirect(reflect.NewValue(data)) + size := TotalSize(v) + if size < 0 { + return os.NewError("binary.Write: invalid type " + v.Type().String()) + } + buf := make([]byte, size) + e := &encoder{order: order, buf: buf} + e.value(v) + _, err := w.Write(buf) + return err +} + +func TotalSize(v reflect.Value) int { + if sv, ok := v.(*reflect.SliceValue); ok { + elem := sizeof(v.Type().(*reflect.SliceType).Elem()) + if elem < 0 { + return -1 + } + return sv.Len() * elem + } + return sizeof(v.Type()) +} + +func sizeof(v reflect.Type) int { + switch t := v.(type) { + case *reflect.ArrayType: + n := sizeof(t.Elem()) + if n < 0 { + return -1 + } + return t.Len() * n + + case *reflect.StructType: + sum := 0 + for i, n := 0, t.NumField(); i < n; i++ { + s := sizeof(t.Field(i).Type) + if s < 0 { + return -1 + } + sum += s + } + return sum + + case *reflect.UintType, *reflect.IntType, *reflect.FloatType, *reflect.ComplexType: + switch t := t.Kind(); t { + case reflect.Int, reflect.Uint, reflect.Uintptr: + return -1 + } + return int(v.Size()) + } + return -1 +} + +type decoder struct { + order ByteOrder + buf []byte +} + +type encoder struct { + order ByteOrder + buf []byte +} + +func (d *decoder) uint8() uint8 { + x := d.buf[0] + d.buf = d.buf[1:] + return x +} + +func (e *encoder) uint8(x uint8) { + e.buf[0] = x + e.buf = e.buf[1:] +} + +func (d *decoder) uint16() uint16 { + x := d.order.Uint16(d.buf[0:2]) + d.buf = d.buf[2:] + return x +} + +func (e *encoder) uint16(x uint16) { + e.order.PutUint16(e.buf[0:2], x) + e.buf = e.buf[2:] +} + +func (d *decoder) uint32() uint32 { + x := d.order.Uint32(d.buf[0:4]) + d.buf = d.buf[4:] + return x +} + +func (e *encoder) uint32(x uint32) { + e.order.PutUint32(e.buf[0:4], x) + e.buf = e.buf[4:] +} + +func (d *decoder) uint64() uint64 { + x := d.order.Uint64(d.buf[0:8]) + d.buf = d.buf[8:] + return x +} + +func (e *encoder) uint64(x uint64) { + e.order.PutUint64(e.buf[0:8], x) + e.buf = e.buf[8:] +} + +func (d *decoder) int8() int8 { return int8(d.uint8()) } + +func (e *encoder) int8(x int8) { e.uint8(uint8(x)) } + +func (d *decoder) int16() int16 { return int16(d.uint16()) } + +func (e *encoder) int16(x int16) { e.uint16(uint16(x)) } + +func (d *decoder) int32() int32 { return int32(d.uint32()) } + +func (e *encoder) int32(x int32) { e.uint32(uint32(x)) } + +func (d *decoder) int64() int64 { return int64(d.uint64()) } + +func (e *encoder) int64(x int64) { e.uint64(uint64(x)) } + +func (d *decoder) value(v reflect.Value) { + switch v := v.(type) { + case *reflect.ArrayValue: + l := v.Len() + for i := 0; i < l; i++ { + d.value(v.Elem(i)) + } + case *reflect.StructValue: + l := v.NumField() + for i := 0; i < l; i++ { + d.value(v.Field(i)) + } + + case *reflect.SliceValue: + l := v.Len() + for i := 0; i < l; i++ { + d.value(v.Elem(i)) + } + + case *reflect.IntValue: + switch v.Type().Kind() { + case reflect.Int8: + v.Set(int64(d.int8())) + case reflect.Int16: + v.Set(int64(d.int16())) + case reflect.Int32: + v.Set(int64(d.int32())) + case reflect.Int64: + v.Set(d.int64()) + } + + case *reflect.UintValue: + switch v.Type().Kind() { + case reflect.Uint8: + v.Set(uint64(d.uint8())) + case reflect.Uint16: + v.Set(uint64(d.uint16())) + case reflect.Uint32: + v.Set(uint64(d.uint32())) + case reflect.Uint64: + v.Set(d.uint64()) + } + + case *reflect.FloatValue: + switch v.Type().Kind() { + case reflect.Float32: + v.Set(float64(math.Float32frombits(d.uint32()))) + case reflect.Float64: + v.Set(math.Float64frombits(d.uint64())) + } + + case *reflect.ComplexValue: + switch v.Type().Kind() { + case reflect.Complex64: + v.Set(complex( + float64(math.Float32frombits(d.uint32())), + float64(math.Float32frombits(d.uint32())), + )) + case reflect.Complex128: + v.Set(complex( + math.Float64frombits(d.uint64()), + math.Float64frombits(d.uint64()), + )) + } + } +} + +func (e *encoder) value(v reflect.Value) { + switch v := v.(type) { + case *reflect.ArrayValue: + l := v.Len() + for i := 0; i < l; i++ { + e.value(v.Elem(i)) + } + case *reflect.StructValue: + l := v.NumField() + for i := 0; i < l; i++ { + e.value(v.Field(i)) + } + case *reflect.SliceValue: + l := v.Len() + for i := 0; i < l; i++ { + e.value(v.Elem(i)) + } + + case *reflect.IntValue: + switch v.Type().Kind() { + case reflect.Int8: + e.int8(int8(v.Get())) + case reflect.Int16: + e.int16(int16(v.Get())) + case reflect.Int32: + e.int32(int32(v.Get())) + case reflect.Int64: + e.int64(v.Get()) + } + + case *reflect.UintValue: + switch v.Type().Kind() { + case reflect.Uint8: + e.uint8(uint8(v.Get())) + case reflect.Uint16: + e.uint16(uint16(v.Get())) + case reflect.Uint32: + e.uint32(uint32(v.Get())) + case reflect.Uint64: + e.uint64(v.Get()) + } + + case *reflect.FloatValue: + switch v.Type().Kind() { + case reflect.Float32: + e.uint32(math.Float32bits(float32(v.Get()))) + case reflect.Float64: + e.uint64(math.Float64bits(v.Get())) + } + + case *reflect.ComplexValue: + switch v.Type().Kind() { + case reflect.Complex64: + x := v.Get() + e.uint32(math.Float32bits(float32(real(x)))) + e.uint32(math.Float32bits(float32(imag(x)))) + case reflect.Complex128: + x := v.Get() + e.uint64(math.Float64bits(real(x))) + e.uint64(math.Float64bits(imag(x))) + } + } +} diff --git a/libgo/go/encoding/binary/binary_test.go b/libgo/go/encoding/binary/binary_test.go new file mode 100644 index 000000000..e09ec489f --- /dev/null +++ b/libgo/go/encoding/binary/binary_test.go @@ -0,0 +1,162 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package binary + +import ( + "os" + "bytes" + "math" + "reflect" + "testing" +) + +type Struct struct { + Int8 int8 + Int16 int16 + Int32 int32 + Int64 int64 + Uint8 uint8 + Uint16 uint16 + Uint32 uint32 + Uint64 uint64 + Float32 float32 + Float64 float64 + Complex64 complex64 + Complex128 complex128 + Array [4]uint8 +} + +type T struct { + Int int + Uint uint + Uintptr uintptr + Array [4]int +} + +var s = Struct{ + 0x01, + 0x0203, + 0x04050607, + 0x08090a0b0c0d0e0f, + 0x10, + 0x1112, + 0x13141516, + 0x1718191a1b1c1d1e, + + math.Float32frombits(0x1f202122), + math.Float64frombits(0x232425262728292a), + complex( + math.Float32frombits(0x2b2c2d2e), + math.Float32frombits(0x2f303132), + ), + complex( + math.Float64frombits(0x333435363738393a), + math.Float64frombits(0x3b3c3d3e3f404142), + ), + + [4]uint8{0x43, 0x44, 0x45, 0x46}, +} + +var big = []byte{ + 1, + 2, 3, + 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, + 17, 18, + 19, 20, 21, 22, + 23, 24, 25, 26, 27, 28, 29, 30, + + 31, 32, 33, 34, + 35, 36, 37, 38, 39, 40, 41, 42, + 43, 44, 45, 46, 47, 48, 49, 50, + 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, + + 67, 68, 69, 70, +} + +var little = []byte{ + 1, + 3, 2, + 7, 6, 5, 4, + 15, 14, 13, 12, 11, 10, 9, 8, + 16, + 18, 17, + 22, 21, 20, 19, + 30, 29, 28, 27, 26, 25, 24, 23, + + 34, 33, 32, 31, + 42, 41, 40, 39, 38, 37, 36, 35, + 46, 45, 44, 43, 50, 49, 48, 47, + 58, 57, 56, 55, 54, 53, 52, 51, 66, 65, 64, 63, 62, 61, 60, 59, + + 67, 68, 69, 70, +} + +var src = []byte{1, 2, 3, 4, 5, 6, 7, 8} +var res = []int32{0x01020304, 0x05060708} + +func checkResult(t *testing.T, dir string, order, err os.Error, have, want interface{}) { + if err != nil { + t.Errorf("%v %v: %v", dir, order, err) + return + } + if !reflect.DeepEqual(have, want) { + t.Errorf("%v %v:\n\thave %+v\n\twant %+v", dir, order, have, want) + } +} + +func testRead(t *testing.T, order ByteOrder, b []byte, s1 interface{}) { + var s2 Struct + err := Read(bytes.NewBuffer(b), order, &s2) + checkResult(t, "Read", order, err, s2, s1) +} + +func testWrite(t *testing.T, order ByteOrder, b []byte, s1 interface{}) { + buf := new(bytes.Buffer) + err := Write(buf, order, s1) + checkResult(t, "Write", order, err, buf.Bytes(), b) +} + +func TestBigEndianRead(t *testing.T) { testRead(t, BigEndian, big, s) } + +func TestLittleEndianRead(t *testing.T) { testRead(t, LittleEndian, little, s) } + +func TestBigEndianWrite(t *testing.T) { testWrite(t, BigEndian, big, s) } + +func TestLittleEndianWrite(t *testing.T) { testWrite(t, LittleEndian, little, s) } + +func TestBigEndianPtrWrite(t *testing.T) { testWrite(t, BigEndian, big, &s) } + +func TestLittleEndianPtrWrite(t *testing.T) { testWrite(t, LittleEndian, little, &s) } + +func TestReadSlice(t *testing.T) { + slice := make([]int32, 2) + err := Read(bytes.NewBuffer(src), BigEndian, slice) + checkResult(t, "ReadSlice", BigEndian, err, slice, res) +} + +func TestWriteSlice(t *testing.T) { + buf := new(bytes.Buffer) + err := Write(buf, BigEndian, res) + checkResult(t, "WriteSlice", BigEndian, err, buf.Bytes(), src) +} + +func TestWriteT(t *testing.T) { + buf := new(bytes.Buffer) + ts := T{} + err := Write(buf, BigEndian, ts) + if err == nil { + t.Errorf("WriteT: have nil, want non-nil") + } + + tv := reflect.Indirect(reflect.NewValue(ts)).(*reflect.StructValue) + for i, n := 0, tv.NumField(); i < n; i++ { + err = Write(buf, BigEndian, tv.Field(i).Interface()) + if err == nil { + t.Errorf("WriteT.%v: have nil, want non-nil", tv.Field(i).Type()) + } + } +} diff --git a/libgo/go/encoding/git85/git.go b/libgo/go/encoding/git85/git.go new file mode 100644 index 000000000..09a45cd3c --- /dev/null +++ b/libgo/go/encoding/git85/git.go @@ -0,0 +1,277 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package git85 implements the radix 85 data encoding +// used in the Git version control system. +package git85 + +import ( + "bytes" + "io" + "os" + "strconv" +) + +type CorruptInputError int64 + +func (e CorruptInputError) String() string { + return "illegal git85 data at input byte " + strconv.Itoa64(int64(e)) +} + +const encode = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!#$%&()*+-;<=>?@^_`{|}~" + +// The decodings are 1+ the actual value, so that the +// default zero value can be used to mean "not valid". +var decode = [256]uint8{ + '0': 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, + 'A': 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, + 'a': 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, + 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, + '!': 63, + '#': 64, 65, 66, 67, + '(': 68, 69, 70, 71, + '-': 72, + ';': 73, + '<': 74, 75, 76, 77, + '@': 78, + '^': 79, 80, 81, + '{': 82, 83, 84, 85, +} + +// Encode encodes src into EncodedLen(len(src)) +// bytes of dst. As a convenience, it returns the number +// of bytes written to dst, but this value is always EncodedLen(len(src)). +// Encode implements the radix 85 encoding used in the +// Git version control tool. +// +// The encoding splits src into chunks of at most 52 bytes +// and encodes each chunk on its own line. +func Encode(dst, src []byte) int { + ndst := 0 + for len(src) > 0 { + n := len(src) + if n > 52 { + n = 52 + } + if n <= 27 { + dst[ndst] = byte('A' + n - 1) + } else { + dst[ndst] = byte('a' + n - 26 - 1) + } + ndst++ + for i := 0; i < n; i += 4 { + var v uint32 + for j := 0; j < 4 && i+j < n; j++ { + v |= uint32(src[i+j]) << uint(24-j*8) + } + for j := 4; j >= 0; j-- { + dst[ndst+j] = encode[v%85] + v /= 85 + } + ndst += 5 + } + dst[ndst] = '\n' + ndst++ + src = src[n:] + } + return ndst +} + +// EncodedLen returns the length of an encoding of n source bytes. +func EncodedLen(n int) int { + if n == 0 { + return 0 + } + // 5 bytes per 4 bytes of input, rounded up. + // 2 extra bytes for each line of 52 src bytes, rounded up. + return (n+3)/4*5 + (n+51)/52*2 +} + +var newline = []byte{'\n'} + +// Decode decodes src into at most MaxDecodedLen(len(src)) +// bytes, returning the actual number of bytes written to dst. +// +// If Decode encounters invalid input, it returns a CorruptInputError. +// +func Decode(dst, src []byte) (n int, err os.Error) { + ndst := 0 + nsrc := 0 + for nsrc < len(src) { + var l int + switch ch := int(src[nsrc]); { + case 'A' <= ch && ch <= 'Z': + l = ch - 'A' + 1 + case 'a' <= ch && ch <= 'z': + l = ch - 'a' + 26 + 1 + default: + return ndst, CorruptInputError(nsrc) + } + if nsrc+1+l > len(src) { + return ndst, CorruptInputError(nsrc) + } + el := (l + 3) / 4 * 5 // encoded len + if nsrc+1+el+1 > len(src) || src[nsrc+1+el] != '\n' { + return ndst, CorruptInputError(nsrc) + } + line := src[nsrc+1 : nsrc+1+el] + for i := 0; i < el; i += 5 { + var v uint32 + for j := 0; j < 5; j++ { + ch := decode[line[i+j]] + if ch == 0 { + return ndst, CorruptInputError(nsrc + 1 + i + j) + } + v = v*85 + uint32(ch-1) + } + for j := 0; j < 4; j++ { + dst[ndst] = byte(v >> 24) + v <<= 8 + ndst++ + } + } + // Last fragment may have run too far (but there was room in dst). + // Back up. + if l%4 != 0 { + ndst -= 4 - l%4 + } + nsrc += 1 + el + 1 + } + return ndst, nil +} + +func MaxDecodedLen(n int) int { return n / 5 * 4 } + +// NewEncoder returns a new Git base85 stream encoder. Data written to +// the returned writer will be encoded and then written to w. +// The Git encoding operates on 52-byte blocks; when finished +// writing, the caller must Close the returned encoder to flush any +// partially written blocks. +func NewEncoder(w io.Writer) io.WriteCloser { return &encoder{w: w} } + +type encoder struct { + w io.Writer + err os.Error + buf [52]byte + nbuf int + out [1024]byte + nout int +} + +func (e *encoder) Write(p []byte) (n int, err os.Error) { + if e.err != nil { + return 0, e.err + } + + // Leading fringe. + if e.nbuf > 0 { + var i int + for i = 0; i < len(p) && e.nbuf < 52; i++ { + e.buf[e.nbuf] = p[i] + e.nbuf++ + } + n += i + p = p[i:] + if e.nbuf < 52 { + return + } + nout := Encode(e.out[0:], e.buf[0:]) + if _, e.err = e.w.Write(e.out[0:nout]); e.err != nil { + return n, e.err + } + e.nbuf = 0 + } + + // Large interior chunks. + for len(p) >= 52 { + nn := len(e.out) / (1 + 52/4*5 + 1) * 52 + if nn > len(p) { + nn = len(p) / 52 * 52 + } + if nn > 0 { + nout := Encode(e.out[0:], p[0:nn]) + if _, e.err = e.w.Write(e.out[0:nout]); e.err != nil { + return n, e.err + } + } + n += nn + p = p[nn:] + } + + // Trailing fringe. + for i := 0; i < len(p); i++ { + e.buf[i] = p[i] + } + e.nbuf = len(p) + n += len(p) + return +} + +func (e *encoder) Close() os.Error { + // If there's anything left in the buffer, flush it out + if e.err == nil && e.nbuf > 0 { + nout := Encode(e.out[0:], e.buf[0:e.nbuf]) + e.nbuf = 0 + _, e.err = e.w.Write(e.out[0:nout]) + } + return e.err +} + +// NewDecoder returns a new Git base85 stream decoder. +func NewDecoder(r io.Reader) io.Reader { return &decoder{r: r} } + +type decoder struct { + r io.Reader + err os.Error + readErr os.Error + buf [1024]byte + nbuf int + out []byte + outbuf [1024]byte + off int64 +} + +func (d *decoder) Read(p []byte) (n int, err os.Error) { + if len(p) == 0 { + return 0, nil + } + + for { + // Copy leftover output from last decode. + if len(d.out) > 0 { + n = copy(p, d.out) + d.out = d.out[n:] + return + } + + // Out of decoded output. Check errors. + if d.err != nil { + return 0, d.err + } + if d.readErr != nil { + d.err = d.readErr + return 0, d.err + } + + // Read and decode more input. + var nn int + nn, d.readErr = d.r.Read(d.buf[d.nbuf:]) + d.nbuf += nn + + // Send complete lines to Decode. + nl := bytes.LastIndex(d.buf[0:d.nbuf], newline) + if nl < 0 { + continue + } + nn, d.err = Decode(d.outbuf[0:], d.buf[0:nl+1]) + if e, ok := d.err.(CorruptInputError); ok { + d.err = CorruptInputError(int64(e) + d.off) + } + d.out = d.outbuf[0:nn] + d.nbuf = copy(d.buf[0:], d.buf[nl+1:d.nbuf]) + d.off += int64(nl + 1) + } + panic("unreacahable") +} diff --git a/libgo/go/encoding/git85/git_test.go b/libgo/go/encoding/git85/git_test.go new file mode 100644 index 000000000..c76385c35 --- /dev/null +++ b/libgo/go/encoding/git85/git_test.go @@ -0,0 +1,194 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package git85 + +import ( + "bytes" + "io/ioutil" + "os" + "testing" +) + +type testpair struct { + decoded, encoded string +} + +func testEqual(t *testing.T, msg string, args ...interface{}) bool { + if args[len(args)-2] != args[len(args)-1] { + t.Errorf(msg, args...) + return false + } + return true +} + +func TestGitTable(t *testing.T) { + var saw [256]bool + for i, c := range encode { + if decode[c] != uint8(i+1) { + t.Errorf("decode['%c'] = %d, want %d", c, decode[c], i+1) + } + saw[c] = true + } + for i, b := range saw { + if !b && decode[i] != 0 { + t.Errorf("decode[%d] = %d, want 0", i, decode[i]) + } + } +} + +var gitPairs = []testpair{ + // Wikipedia example, adapted. + { + "Man is distinguished, not only by his reason, but by this singular passion from " + + "other animals, which is a lust of the mind, that by a perseverance of delight in " + + "the continued and indefatigable generation of knowledge, exceeds the short " + + "vehemence of any carnal pleasure.", + + "zO<`^zX>%ZCX>)XGZfA9Ab7*B`EFf-gbRchTYDO_b1WctXlY|;AZc?T\n" + + "zVIXXEb95kYW*~HEWgu;7Ze%PVbZB98AYyqSVIXj2a&u*NWpZI|V`U(3W*}r`Y-wj`\n" + + "zbRcPNAarPDAY*TCbZKsNWn>^>Ze$>7Ze(RV>IZ)PBC\n" + + "zZf|#NWn^b%EFfigV`XJzb0BnRWgv5CZ*p`Xc4cT~ZDnp_Wgu^6AYpEKAY);2ZeeU7\n" + + "IaBO8^b9HiME&u=k\n", + }, +} + +var gitBigtest = gitPairs[len(gitPairs)-1] + +func TestEncode(t *testing.T) { + for _, p := range gitPairs { + buf := make([]byte, EncodedLen(len(p.decoded))) + n := Encode(buf, []byte(p.decoded)) + if n != len(buf) { + t.Errorf("EncodedLen does not agree with Encode") + } + buf = buf[0:n] + testEqual(t, "Encode(%q) = %q, want %q", p.decoded, string(buf), p.encoded) + } +} + +func TestEncoder(t *testing.T) { + for _, p := range gitPairs { + bb := &bytes.Buffer{} + encoder := NewEncoder(bb) + encoder.Write([]byte(p.decoded)) + encoder.Close() + testEqual(t, "Encode(%q) = %q, want %q", p.decoded, bb.String(), p.encoded) + } +} + +func TestEncoderBuffering(t *testing.T) { + input := []byte(gitBigtest.decoded) + for bs := 1; bs <= 12; bs++ { + bb := &bytes.Buffer{} + encoder := NewEncoder(bb) + for pos := 0; pos < len(input); pos += bs { + end := pos + bs + if end > len(input) { + end = len(input) + } + n, err := encoder.Write(input[pos:end]) + testEqual(t, "Write(%q) gave error %v, want %v", input[pos:end], err, os.Error(nil)) + testEqual(t, "Write(%q) gave length %v, want %v", input[pos:end], n, end-pos) + } + err := encoder.Close() + testEqual(t, "Close gave error %v, want %v", err, os.Error(nil)) + testEqual(t, "Encoding/%d of %q = %q, want %q", bs, gitBigtest.decoded, bb.String(), gitBigtest.encoded) + } +} + +func TestDecode(t *testing.T) { + for _, p := range gitPairs { + dbuf := make([]byte, 4*len(p.encoded)) + ndst, err := Decode(dbuf, []byte(p.encoded)) + testEqual(t, "Decode(%q) = error %v, want %v", p.encoded, err, os.Error(nil)) + testEqual(t, "Decode(%q) = ndst %v, want %v", p.encoded, ndst, len(p.decoded)) + testEqual(t, "Decode(%q) = %q, want %q", p.encoded, string(dbuf[0:ndst]), p.decoded) + } +} + +func TestDecoder(t *testing.T) { + for _, p := range gitPairs { + decoder := NewDecoder(bytes.NewBufferString(p.encoded)) + dbuf, err := ioutil.ReadAll(decoder) + if err != nil { + t.Fatal("Read failed", err) + } + testEqual(t, "Read from %q = length %v, want %v", p.encoded, len(dbuf), len(p.decoded)) + testEqual(t, "Decoding of %q = %q, want %q", p.encoded, string(dbuf), p.decoded) + if err != nil { + testEqual(t, "Read from %q = %v, want %v", p.encoded, err, os.EOF) + } + } +} + +func TestDecoderBuffering(t *testing.T) { + for bs := 1; bs <= 12; bs++ { + decoder := NewDecoder(bytes.NewBufferString(gitBigtest.encoded)) + buf := make([]byte, len(gitBigtest.decoded)+12) + var total int + for total = 0; total < len(gitBigtest.decoded); { + n, err := decoder.Read(buf[total : total+bs]) + testEqual(t, "Read from %q at pos %d = %d, %v, want _, %v", gitBigtest.encoded, total, n, err, os.Error(nil)) + total += n + } + testEqual(t, "Decoding/%d of %q = %q, want %q", bs, gitBigtest.encoded, string(buf[0:total]), gitBigtest.decoded) + } +} + +func TestDecodeCorrupt(t *testing.T) { + type corrupt struct { + e string + p int + } + examples := []corrupt{ + {"v", 0}, + {"!z!!!!!!!!!", 0}, + } + + for _, e := range examples { + dbuf := make([]byte, 2*len(e.e)) + _, err := Decode(dbuf, []byte(e.e)) + switch err := err.(type) { + case CorruptInputError: + testEqual(t, "Corruption in %q at offset %v, want %v", e.e, int(err), e.p) + default: + t.Error("Decoder failed to detect corruption in", e) + } + } +} + +func TestGitBig(t *testing.T) { + n := 3*1000 + 1 + raw := make([]byte, n) + const alpha = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + for i := 0; i < n; i++ { + raw[i] = alpha[i%len(alpha)] + } + encoded := new(bytes.Buffer) + w := NewEncoder(encoded) + nn, err := w.Write(raw) + if nn != n || err != nil { + t.Fatalf("Encoder.Write(raw) = %d, %v want %d, nil", nn, err, n) + } + err = w.Close() + if err != nil { + t.Fatalf("Encoder.Close() = %v want nil", err) + } + decoded, err := ioutil.ReadAll(NewDecoder(encoded)) + if err != nil { + t.Fatalf("ioutil.ReadAll(NewDecoder(...)): %v", err) + } + + if !bytes.Equal(raw, decoded) { + var i int + for i = 0; i < len(decoded) && i < len(raw); i++ { + if decoded[i] != raw[i] { + break + } + } + t.Errorf("Decode(Encode(%d-byte string)) failed at offset %d", n, i) + } +} diff --git a/libgo/go/encoding/hex/hex.go b/libgo/go/encoding/hex/hex.go new file mode 100644 index 000000000..292d917eb --- /dev/null +++ b/libgo/go/encoding/hex/hex.go @@ -0,0 +1,101 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This package implements hexadecimal encoding and decoding. +package hex + +import ( + "os" + "strconv" +) + +const hextable = "0123456789abcdef" + +// EncodedLen returns the length of an encoding of n source bytes. +func EncodedLen(n int) int { return n * 2 } + +// Encode encodes src into EncodedLen(len(src)) +// bytes of dst. As a convenience, it returns the number +// of bytes written to dst, but this value is always EncodedLen(len(src)). +// Encode implements hexadecimal encoding. +func Encode(dst, src []byte) int { + for i, v := range src { + dst[i*2] = hextable[v>>4] + dst[i*2+1] = hextable[v&0x0f] + } + + return len(src) * 2 +} + +// OddLengthInputError results from decoding an odd length slice. +type OddLengthInputError struct{} + +func (OddLengthInputError) String() string { return "odd length hex string" } + +// InvalidHexCharError results from finding an invalid character in a hex string. +type InvalidHexCharError byte + +func (e InvalidHexCharError) String() string { + return "invalid hex char: " + strconv.Itoa(int(e)) +} + + +func DecodedLen(x int) int { return x / 2 } + +// Decode decodes src into DecodedLen(len(src)) bytes, returning the actual +// number of bytes written to dst. +// +// If Decode encounters invalid input, it returns an OddLengthInputError or an +// InvalidHexCharError. +func Decode(dst, src []byte) (int, os.Error) { + if len(src)%2 == 1 { + return 0, OddLengthInputError{} + } + + for i := 0; i < len(src)/2; i++ { + a, ok := fromHexChar(src[i*2]) + if !ok { + return 0, InvalidHexCharError(src[i*2]) + } + b, ok := fromHexChar(src[i*2+1]) + if !ok { + return 0, InvalidHexCharError(src[i*2+1]) + } + dst[i] = (a << 4) | b + } + + return len(src) / 2, nil +} + +// fromHexChar converts a hex character into its value and a success flag. +func fromHexChar(c byte) (byte, bool) { + switch { + case '0' <= c && c <= '9': + return c - '0', true + case 'a' <= c && c <= 'f': + return c - 'a' + 10, true + case 'A' <= c && c <= 'F': + return c - 'A' + 10, true + } + + return 0, false +} + +// EncodeToString returns the hexadecimal encoding of src. +func EncodeToString(src []byte) string { + dst := make([]byte, EncodedLen(len(src))) + Encode(dst, src) + return string(dst) +} + +// DecodeString returns the bytes represented by the hexadecimal string s. +func DecodeString(s string) ([]byte, os.Error) { + src := []byte(s) + dst := make([]byte, DecodedLen(len(src))) + _, err := Decode(dst, src) + if err != nil { + return nil, err + } + return dst, nil +} diff --git a/libgo/go/encoding/hex/hex_test.go b/libgo/go/encoding/hex/hex_test.go new file mode 100644 index 000000000..a14c9d4f4 --- /dev/null +++ b/libgo/go/encoding/hex/hex_test.go @@ -0,0 +1,149 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package hex + +import ( + "bytes" + "testing" +) + +type encodeTest struct { + in, out []byte +} + +var encodeTests = []encodeTest{ + {[]byte{}, []byte{}}, + {[]byte{0x01}, []byte{'0', '1'}}, + {[]byte{0xff}, []byte{'f', 'f'}}, + {[]byte{0xff, 00}, []byte{'f', 'f', '0', '0'}}, + {[]byte{0}, []byte{'0', '0'}}, + {[]byte{1}, []byte{'0', '1'}}, + {[]byte{2}, []byte{'0', '2'}}, + {[]byte{3}, []byte{'0', '3'}}, + {[]byte{4}, []byte{'0', '4'}}, + {[]byte{5}, []byte{'0', '5'}}, + {[]byte{6}, []byte{'0', '6'}}, + {[]byte{7}, []byte{'0', '7'}}, + {[]byte{8}, []byte{'0', '8'}}, + {[]byte{9}, []byte{'0', '9'}}, + {[]byte{10}, []byte{'0', 'a'}}, + {[]byte{11}, []byte{'0', 'b'}}, + {[]byte{12}, []byte{'0', 'c'}}, + {[]byte{13}, []byte{'0', 'd'}}, + {[]byte{14}, []byte{'0', 'e'}}, + {[]byte{15}, []byte{'0', 'f'}}, +} + +func TestEncode(t *testing.T) { + for i, test := range encodeTests { + dst := make([]byte, EncodedLen(len(test.in))) + n := Encode(dst, test.in) + if n != len(dst) { + t.Errorf("#%d: bad return value: got: %d want: %d", i, n, len(dst)) + } + if bytes.Compare(dst, test.out) != 0 { + t.Errorf("#%d: got: %#v want: %#v", i, dst, test.out) + } + } +} + +type decodeTest struct { + in, out []byte + ok bool +} + +var decodeTests = []decodeTest{ + {[]byte{}, []byte{}, true}, + {[]byte{'0'}, []byte{}, false}, + {[]byte{'0', 'g'}, []byte{}, false}, + {[]byte{'0', '\x01'}, []byte{}, false}, + {[]byte{'0', '0'}, []byte{0}, true}, + {[]byte{'0', '1'}, []byte{1}, true}, + {[]byte{'0', '2'}, []byte{2}, true}, + {[]byte{'0', '3'}, []byte{3}, true}, + {[]byte{'0', '4'}, []byte{4}, true}, + {[]byte{'0', '5'}, []byte{5}, true}, + {[]byte{'0', '6'}, []byte{6}, true}, + {[]byte{'0', '7'}, []byte{7}, true}, + {[]byte{'0', '8'}, []byte{8}, true}, + {[]byte{'0', '9'}, []byte{9}, true}, + {[]byte{'0', 'a'}, []byte{10}, true}, + {[]byte{'0', 'b'}, []byte{11}, true}, + {[]byte{'0', 'c'}, []byte{12}, true}, + {[]byte{'0', 'd'}, []byte{13}, true}, + {[]byte{'0', 'e'}, []byte{14}, true}, + {[]byte{'0', 'f'}, []byte{15}, true}, + {[]byte{'0', 'A'}, []byte{10}, true}, + {[]byte{'0', 'B'}, []byte{11}, true}, + {[]byte{'0', 'C'}, []byte{12}, true}, + {[]byte{'0', 'D'}, []byte{13}, true}, + {[]byte{'0', 'E'}, []byte{14}, true}, + {[]byte{'0', 'F'}, []byte{15}, true}, +} + +func TestDecode(t *testing.T) { + for i, test := range decodeTests { + dst := make([]byte, DecodedLen(len(test.in))) + n, err := Decode(dst, test.in) + if err == nil && n != len(dst) { + t.Errorf("#%d: bad return value: got:%d want:%d", i, n, len(dst)) + } + if test.ok != (err == nil) { + t.Errorf("#%d: unexpected err value: %s", i, err) + } + if err == nil && bytes.Compare(dst, test.out) != 0 { + t.Errorf("#%d: got: %#v want: %#v", i, dst, test.out) + } + } +} + +type encodeStringTest struct { + in []byte + out string +} + +var encodeStringTests = []encodeStringTest{ + {[]byte{}, ""}, + {[]byte{0}, "00"}, + {[]byte{0, 1}, "0001"}, + {[]byte{0, 1, 255}, "0001ff"}, +} + +func TestEncodeToString(t *testing.T) { + for i, test := range encodeStringTests { + s := EncodeToString(test.in) + if s != test.out { + t.Errorf("#%d got:%s want:%s", i, s, test.out) + } + } +} + +type decodeStringTest struct { + in string + out []byte + ok bool +} + +var decodeStringTests = []decodeStringTest{ + {"", []byte{}, true}, + {"0", []byte{}, false}, + {"00", []byte{0}, true}, + {"0\x01", []byte{}, false}, + {"0g", []byte{}, false}, + {"00ff00", []byte{0, 255, 0}, true}, + {"0000ff", []byte{0, 0, 255}, true}, +} + +func TestDecodeString(t *testing.T) { + for i, test := range decodeStringTests { + dst, err := DecodeString(test.in) + if test.ok != (err == nil) { + t.Errorf("#%d: unexpected err value: %s", i, err) + } + if err == nil && bytes.Compare(dst, test.out) != 0 { + t.Errorf("#%d: got: %#v want: #%v", i, dst, test.out) + } + } +} diff --git a/libgo/go/encoding/line/line.go b/libgo/go/encoding/line/line.go new file mode 100644 index 000000000..92dddcb99 --- /dev/null +++ b/libgo/go/encoding/line/line.go @@ -0,0 +1,95 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This package implements a Reader which handles reading \r and \r\n +// deliminated lines. +package line + +import ( + "io" + "os" +) + +// Reader reads lines from an io.Reader (which may use either '\n' or +// '\r\n'). +type Reader struct { + buf []byte + consumed int + in io.Reader + err os.Error +} + +func NewReader(in io.Reader, maxLineLength int) *Reader { + return &Reader{ + buf: make([]byte, 0, maxLineLength), + consumed: 0, + in: in, + } +} + +// ReadLine tries to return a single line, not including the end-of-line bytes. +// If the line was found to be longer than the maximum length then isPrefix is +// set and the beginning of the line is returned. The rest of the line will be +// returned from future calls. isPrefix will be false when returning the last +// fragment of the line. The returned buffer points into the internal state of +// the Reader and is only valid until the next call to ReadLine. ReadLine +// either returns a non-nil line or it returns an error, never both. +func (l *Reader) ReadLine() (line []byte, isPrefix bool, err os.Error) { + if l.consumed > 0 { + n := copy(l.buf, l.buf[l.consumed:]) + l.buf = l.buf[:n] + l.consumed = 0 + } + + if len(l.buf) == 0 && l.err != nil { + err = l.err + return + } + + scannedTo := 0 + + for { + i := scannedTo + for ; i < len(l.buf); i++ { + if l.buf[i] == '\r' && len(l.buf) > i+1 && l.buf[i+1] == '\n' { + line = l.buf[:i] + l.consumed = i + 2 + return + } else if l.buf[i] == '\n' { + line = l.buf[:i] + l.consumed = i + 1 + return + } + } + + if i == cap(l.buf) { + line = l.buf[:i] + l.consumed = i + isPrefix = true + return + } + + if l.err != nil { + line = l.buf + l.consumed = i + return + } + + // We don't want to rescan the input that we just scanned. + // However, we need to back up one byte because the last byte + // could have been a '\r' and we do need to rescan that. + scannedTo = i + if scannedTo > 0 { + scannedTo-- + } + oldLen := len(l.buf) + l.buf = l.buf[:cap(l.buf)] + n, readErr := l.in.Read(l.buf[oldLen:]) + l.buf = l.buf[:oldLen+n] + if readErr != nil { + l.err = readErr + } + } + panic("unreachable") +} diff --git a/libgo/go/encoding/line/line_test.go b/libgo/go/encoding/line/line_test.go new file mode 100644 index 000000000..68d13b586 --- /dev/null +++ b/libgo/go/encoding/line/line_test.go @@ -0,0 +1,89 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package line + +import ( + "bytes" + "os" + "testing" +) + +var testOutput = []byte("0123456789abcdefghijklmnopqrstuvwxy") +var testInput = []byte("012\n345\n678\n9ab\ncde\nfgh\nijk\nlmn\nopq\nrst\nuvw\nxy") +var testInputrn = []byte("012\r\n345\r\n678\r\n9ab\r\ncde\r\nfgh\r\nijk\r\nlmn\r\nopq\r\nrst\r\nuvw\r\nxy\r\n\n\r\n") + +// TestReader wraps a []byte and returns reads of a specific length. +type testReader struct { + data []byte + stride int +} + +func (t *testReader) Read(buf []byte) (n int, err os.Error) { + n = t.stride + if n > len(t.data) { + n = len(t.data) + } + if n > len(buf) { + n = len(buf) + } + copy(buf, t.data) + t.data = t.data[n:] + if len(t.data) == 0 { + err = os.EOF + } + return +} + +func testLineReader(t *testing.T, input []byte) { + for stride := 1; stride < len(input); stride++ { + done := 0 + reader := testReader{input, stride} + l := NewReader(&reader, len(input)+1) + for { + line, isPrefix, err := l.ReadLine() + if len(line) > 0 && err != nil { + t.Errorf("ReadLine returned both data and error: %s", err) + } + if isPrefix { + t.Errorf("ReadLine returned prefix") + } + if err != nil { + if err != os.EOF { + t.Fatalf("Got unknown error: %s", err) + } + break + } + if want := testOutput[done : done+len(line)]; !bytes.Equal(want, line) { + t.Errorf("Bad line at stride %d: want: %x got: %x", stride, want, line) + } + done += len(line) + } + if done != len(testOutput) { + t.Error("ReadLine didn't return everything") + } + } +} + +func TestReader(t *testing.T) { + testLineReader(t, testInput) + testLineReader(t, testInputrn) +} + +func TestLineTooLong(t *testing.T) { + buf := bytes.NewBuffer([]byte("aaabbbcc\n")) + l := NewReader(buf, 3) + line, isPrefix, err := l.ReadLine() + if !isPrefix || !bytes.Equal(line, []byte("aaa")) || err != nil { + t.Errorf("bad result for first line: %x %s", line, err) + } + line, isPrefix, err = l.ReadLine() + if !isPrefix || !bytes.Equal(line, []byte("bbb")) || err != nil { + t.Errorf("bad result for second line: %x", line) + } + line, isPrefix, err = l.ReadLine() + if isPrefix || !bytes.Equal(line, []byte("cc")) || err != nil { + t.Errorf("bad result for third line: %x", line) + } +} diff --git a/libgo/go/encoding/pem/pem.go b/libgo/go/encoding/pem/pem.go new file mode 100644 index 000000000..5653aeb77 --- /dev/null +++ b/libgo/go/encoding/pem/pem.go @@ -0,0 +1,257 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This package implements the PEM data encoding, which originated in Privacy +// Enhanced Mail. The most common use of PEM encoding today is in TLS keys and +// certificates. See RFC 1421. +package pem + +import ( + "bytes" + "encoding/base64" + "io" + "os" +) + +// A Block represents a PEM encoded structure. +// +// The encoded form is: +// -----BEGIN Type----- +// Headers +// base64-encoded Bytes +// -----END Type----- +// where Headers is a possibly empty sequence of Key: Value lines. +type Block struct { + Type string // The type, taken from the preamble (i.e. "RSA PRIVATE KEY"). + Headers map[string]string // Optional headers. + Bytes []byte // The decoded bytes of the contents. Typically a DER encoded ASN.1 structure. +} + +// getLine results the first \r\n or \n delineated line from the given byte +// array. The line does not include the \r\n or \n. The remainder of the byte +// array (also not including the new line bytes) is also returned and this will +// always be smaller than the original argument. +func getLine(data []byte) (line, rest []byte) { + i := bytes.Index(data, []byte{'\n'}) + var j int + if i < 0 { + i = len(data) + j = i + } else { + j = i + 1 + if i > 0 && data[i-1] == '\r' { + i-- + } + } + return data[0:i], data[j:] +} + +// removeWhitespace returns a copy of its input with all spaces, tab and +// newline characters removed. +func removeWhitespace(data []byte) []byte { + result := make([]byte, len(data)) + n := 0 + + for _, b := range data { + if b == ' ' || b == '\t' || b == '\r' || b == '\n' { + continue + } + result[n] = b + n++ + } + + return result[0:n] +} + +var pemStart = []byte("\n-----BEGIN ") +var pemEnd = []byte("\n-----END ") +var pemEndOfLine = []byte("-----") + +// Decode will find the next PEM formatted block (certificate, private key +// etc) in the input. It returns that block and the remainder of the input. If +// no PEM data is found, p is nil and the whole of the input is returned in +// rest. +func Decode(data []byte) (p *Block, rest []byte) { + // pemStart begins with a newline. However, at the very beginning of + // the byte array, we'll accept the start string without it. + rest = data + if bytes.HasPrefix(data, pemStart[1:]) { + rest = rest[len(pemStart)-1 : len(data)] + } else if i := bytes.Index(data, pemStart); i >= 0 { + rest = rest[i+len(pemStart) : len(data)] + } else { + return nil, data + } + + typeLine, rest := getLine(rest) + if !bytes.HasSuffix(typeLine, pemEndOfLine) { + goto Error + } + typeLine = typeLine[0 : len(typeLine)-len(pemEndOfLine)] + + p = &Block{ + Headers: make(map[string]string), + Type: string(typeLine), + } + + for { + // This loop terminates because getLine's second result is + // always smaller than it's argument. + if len(rest) == 0 { + return nil, data + } + line, next := getLine(rest) + + i := bytes.Index(line, []byte{':'}) + if i == -1 { + break + } + + // TODO(agl): need to cope with values that spread across lines. + key, val := line[0:i], line[i+1:] + key = bytes.TrimSpace(key) + val = bytes.TrimSpace(val) + p.Headers[string(key)] = string(val) + rest = next + } + + i := bytes.Index(rest, pemEnd) + if i < 0 { + goto Error + } + base64Data := removeWhitespace(rest[0:i]) + + p.Bytes = make([]byte, base64.StdEncoding.DecodedLen(len(base64Data))) + n, err := base64.StdEncoding.Decode(p.Bytes, base64Data) + if err != nil { + goto Error + } + p.Bytes = p.Bytes[0:n] + + _, rest = getLine(rest[i+len(pemEnd):]) + + return + +Error: + // If we get here then we have rejected a likely looking, but + // ultimately invalid PEM block. We need to start over from a new + // position. We have consumed the preamble line and will have consumed + // any lines which could be header lines. However, a valid preamble + // line is not a valid header line, therefore we cannot have consumed + // the preamble line for the any subsequent block. Thus, we will always + // find any valid block, no matter what bytes preceed it. + // + // For example, if the input is + // + // -----BEGIN MALFORMED BLOCK----- + // junk that may look like header lines + // or data lines, but no END line + // + // -----BEGIN ACTUAL BLOCK----- + // realdata + // -----END ACTUAL BLOCK----- + // + // we've failed to parse using the first BEGIN line + // and now will try again, using the second BEGIN line. + p, rest = Decode(rest) + if p == nil { + rest = data + } + return +} + +const pemLineLength = 64 + +type lineBreaker struct { + line [pemLineLength]byte + used int + out io.Writer +} + +func (l *lineBreaker) Write(b []byte) (n int, err os.Error) { + if l.used+len(b) < pemLineLength { + copy(l.line[l.used:], b) + l.used += len(b) + return len(b), nil + } + + n, err = l.out.Write(l.line[0:l.used]) + if err != nil { + return + } + excess := pemLineLength - l.used + l.used = 0 + + n, err = l.out.Write(b[0:excess]) + if err != nil { + return + } + + n, err = l.out.Write([]byte{'\n'}) + if err != nil { + return + } + + return l.Write(b[excess:]) +} + +func (l *lineBreaker) Close() (err os.Error) { + if l.used > 0 { + _, err = l.out.Write(l.line[0:l.used]) + if err != nil { + return + } + _, err = l.out.Write([]byte{'\n'}) + } + + return +} + +func Encode(out io.Writer, b *Block) (err os.Error) { + _, err = out.Write(pemStart[1:]) + if err != nil { + return + } + _, err = out.Write([]byte(b.Type + "-----\n")) + if err != nil { + return + } + + if len(b.Headers) > 0 { + for k, v := range b.Headers { + _, err = out.Write([]byte(k + ": " + v + "\n")) + if err != nil { + return + } + } + _, err = out.Write([]byte{'\n'}) + if err != nil { + return + } + } + + var breaker lineBreaker + breaker.out = out + + b64 := base64.NewEncoder(base64.StdEncoding, &breaker) + _, err = b64.Write(b.Bytes) + if err != nil { + return + } + b64.Close() + breaker.Close() + + _, err = out.Write(pemEnd[1:]) + if err != nil { + return + } + _, err = out.Write([]byte(b.Type + "-----\n")) + return +} + +func EncodeToMemory(b *Block) []byte { + buf := bytes.NewBuffer(nil) + Encode(buf, b) + return buf.Bytes() +} diff --git a/libgo/go/encoding/pem/pem_test.go b/libgo/go/encoding/pem/pem_test.go new file mode 100644 index 000000000..11efe5544 --- /dev/null +++ b/libgo/go/encoding/pem/pem_test.go @@ -0,0 +1,390 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package pem + +import ( + "bytes" + "reflect" + "testing" +) + +type GetLineTest struct { + in, out1, out2 string +} + +var getLineTests = []GetLineTest{ + {"abc", "abc", ""}, + {"abc\r", "abc\r", ""}, + {"abc\n", "abc", ""}, + {"abc\r\n", "abc", ""}, + {"abc\nd", "abc", "d"}, + {"abc\r\nd", "abc", "d"}, + {"\nabc", "", "abc"}, + {"\r\nabc", "", "abc"}, +} + +func TestGetLine(t *testing.T) { + for i, test := range getLineTests { + x, y := getLine([]byte(test.in)) + if string(x) != test.out1 || string(y) != test.out2 { + t.Errorf("#%d got:%+v,%+v want:%s,%s", i, x, y, test.out1, test.out2) + } + } +} + +func TestDecode(t *testing.T) { + result, remainder := Decode([]byte(pemData)) + if !reflect.DeepEqual(result, certificate) { + t.Errorf("#0 got:%#v want:%#v", result, certificate) + } + result, remainder = Decode(remainder) + if !reflect.DeepEqual(result, privateKey) { + t.Errorf("#1 got:%#v want:%#v", result, privateKey) + } + result, _ = Decode([]byte(pemPrivateKey)) + if !reflect.DeepEqual(result, privateKey2) { + t.Errorf("#2 got:%#v want:%#v", result, privateKey2) + } +} + +func TestEncode(t *testing.T) { + r := EncodeToMemory(privateKey2) + if string(r) != pemPrivateKey { + t.Errorf("got:%s want:%s", r, pemPrivateKey) + } +} + +type lineBreakerTest struct { + in, out string +} + +const sixtyFourCharString = "0123456789012345678901234567890123456789012345678901234567890123" + +var lineBreakerTests = []lineBreakerTest{ + {"", ""}, + {"a", "a\n"}, + {"ab", "ab\n"}, + {sixtyFourCharString, sixtyFourCharString + "\n"}, + {sixtyFourCharString + "X", sixtyFourCharString + "\nX\n"}, + {sixtyFourCharString + sixtyFourCharString, sixtyFourCharString + "\n" + sixtyFourCharString + "\n"}, +} + +func TestLineBreaker(t *testing.T) { + for i, test := range lineBreakerTests { + buf := bytes.NewBuffer(nil) + var breaker lineBreaker + breaker.out = buf + _, err := breaker.Write([]byte(test.in)) + if err != nil { + t.Errorf("#%d: error from Write: %s", i, err) + continue + } + err = breaker.Close() + if err != nil { + t.Errorf("#%d: error from Close: %s", i, err) + continue + } + + if string(buf.Bytes()) != test.out { + t.Errorf("#%d: got:%s want:%s", i, string(buf.Bytes()), test.out) + } + } + + for i, test := range lineBreakerTests { + buf := bytes.NewBuffer(nil) + var breaker lineBreaker + breaker.out = buf + + for i := 0; i < len(test.in); i++ { + _, err := breaker.Write([]byte(test.in[i : i+1])) + if err != nil { + t.Errorf("#%d: error from Write (byte by byte): %s", i, err) + continue + } + } + err := breaker.Close() + if err != nil { + t.Errorf("#%d: error from Close (byte by byte): %s", i, err) + continue + } + + if string(buf.Bytes()) != test.out { + t.Errorf("#%d: (byte by byte) got:%s want:%s", i, string(buf.Bytes()), test.out) + } + } +} + +var pemData = `verify return:0 +-----BEGIN CERTIFICATE----- +sdlfkjskldfj + -----BEGIN CERTIFICATE----- +--- +Certificate chain + 0 s:/C=AU/ST=Somewhere/L=Someplace/O=Foo Bar/CN=foo.example.com + i:/C=ZA/O=CA Inc./CN=CA Inc +-----BEGIN CERTIFICATE----- +testing +-----BEGIN CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIID6TCCA1ICAQEwDQYJKoZIhvcNAQEFBQAwgYsxCzAJBgNVBAYTAlVTMRMwEQYD +VQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQK +EwtHb29nbGUgSW5jLjEMMAoGA1UECxMDRW5nMQwwCgYDVQQDEwNhZ2wxHTAbBgkq +hkiG9w0BCQEWDmFnbEBnb29nbGUuY29tMB4XDTA5MDkwOTIyMDU0M1oXDTEwMDkw +OTIyMDU0M1owajELMAkGA1UEBhMCQVUxEzARBgNVBAgTClNvbWUtU3RhdGUxITAf +BgNVBAoTGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEjMCEGA1UEAxMaZXVyb3Bh +LnNmby5jb3JwLmdvb2dsZS5jb20wggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK +AoICAQC6pgYt7/EibBDumASF+S0qvqdL/f+nouJw2T1Qc8GmXF/iiUcrsgzh/Fd8 +pDhz/T96Qg9IyR4ztuc2MXrmPra+zAuSf5bevFReSqvpIt8Duv0HbDbcqs/XKPfB +uMDe+of7a9GCywvAZ4ZUJcp0thqD9fKTTjUWOBzHY1uNE4RitrhmJCrbBGXbJ249 +bvgmb7jgdInH2PU7PT55hujvOoIsQW2osXBFRur4pF1wmVh4W4lTLD6pjfIMUcML +ICHEXEN73PDic8KS3EtNYCwoIld+tpIBjE1QOb1KOyuJBNW6Esw9ALZn7stWdYcE +qAwvv20egN2tEXqj7Q4/1ccyPZc3PQgC3FJ8Be2mtllM+80qf4dAaQ/fWvCtOrQ5 +pnfe9juQvCo8Y0VGlFcrSys/MzSg9LJ/24jZVgzQved/Qupsp89wVidwIzjt+WdS +fyWfH0/v1aQLvu5cMYuW//C0W2nlYziL5blETntM8My2ybNARy3ICHxCBv2RNtPI +WQVm+E9/W5rwh2IJR4DHn2LHwUVmT/hHNTdBLl5Uhwr4Wc7JhE7AVqb14pVNz1lr +5jxsp//ncIwftb7mZQ3DF03Yna+jJhpzx8CQoeLT6aQCHyzmH68MrHHT4MALPyUs +Pomjn71GNTtDeWAXibjCgdL6iHACCF6Htbl0zGlG0OAK+bdn0QIDAQABMA0GCSqG +SIb3DQEBBQUAA4GBAOKnQDtqBV24vVqvesL5dnmyFpFPXBn3WdFfwD6DzEb21UVG +5krmJiu+ViipORJPGMkgoL6BjU21XI95VQbun5P8vvg8Z+FnFsvRFY3e1CCzAVQY +ZsUkLw2I7zI/dNlWdB8Xp7v+3w9sX5N3J/WuJ1KOO5m26kRlHQo7EzT3974g +-----END CERTIFICATE----- + 1 s:/C=ZA/O=Ca Inc./CN=CA Inc + +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: DES-EDE3-CBC,80C7C7A09690757A + +eQp5ZkH6CyHBz7BZfUPxyLCCmftsBJ7HlqGb8Ld21cSwnzWZ4/SIlhyrUtsfw7VR +2TTwA+odo9ex7GdxOTaH8oZFumIRoiEjHsk8U7Bhntp+ekkPP79xunnN7hb7hkhr +yGDQZgA7s2cQHQ71v3gwT2BACAft26jCjbM1wgNzBnJ8M0Rzn68YWqaPtdBu8qb/ +zVR5JB1mnqvTSbFsfF5yMc6o2WQ9jJCl6KypnMl+BpL+dlvdjYVK4l9lYsB1Hs3d ++zDBbWxos818zzhS8/y6eIfiSG27cqrbhURbmgiSfDXjncK4m/pLcQ7mmBL6mFOr +3Pj4jepzgOiFRL6MKE//h62fZvI1ErYr8VunHEykgKNhChDvb1RO6LEfqKBu+Ivw +TB6fBhW3TCLMnVPYVoYwA+fHNTmZZm8BEonlIMfI+KktjWUg4Oia+NI6vKcPpFox +hSnlGgCtvfEaq5/H4kHJp95eOpnFsLviw2seHNkz/LxJMRP1X428+DpYW/QD/0JU +tJSuC/q9FUHL6RI3u/Asrv8pCb4+D7i1jW/AMIdJTtycOGsbPxQA7yHMWujHmeb1 +BTiHcL3s3KrJu1vDVrshvxfnz71KTeNnZH8UbOqT5i7fPGyXtY1XJddcbI/Q6tXf +wHFsZc20TzSdsVLBtwksUacpbDogcEVMctnNrB8FIrB3vZEv9Q0Z1VeY7nmTpF+6 +a+z2P7acL7j6A6Pr3+q8P9CPiPC7zFonVzuVPyB8GchGR2hytyiOVpuD9+k8hcuw +ZWAaUoVtWIQ52aKS0p19G99hhb+IVANC4akkdHV4SP8i7MVNZhfUmg== +-----END RSA PRIVATE KEY-----` + +var certificate = &Block{Type: "CERTIFICATE", + Headers: map[string]string{}, + Bytes: []uint8{0x30, 0x82, 0x3, 0xe9, 0x30, 0x82, 0x3, 0x52, 0x2, 0x1, + 0x1, 0x30, 0xd, 0x6, 0x9, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0xd, + 0x1, 0x1, 0x5, 0x5, 0x0, 0x30, 0x81, 0x8b, 0x31, 0xb, 0x30, + 0x9, 0x6, 0x3, 0x55, 0x4, 0x6, 0x13, 0x2, 0x55, 0x53, 0x31, + 0x13, 0x30, 0x11, 0x6, 0x3, 0x55, 0x4, 0x8, 0x13, 0xa, 0x43, + 0x61, 0x6c, 0x69, 0x66, 0x6f, 0x72, 0x6e, 0x69, 0x61, 0x31, + 0x16, 0x30, 0x14, 0x6, 0x3, 0x55, 0x4, 0x7, 0x13, 0xd, 0x53, + 0x61, 0x6e, 0x20, 0x46, 0x72, 0x61, 0x6e, 0x63, 0x69, 0x73, + 0x63, 0x6f, 0x31, 0x14, 0x30, 0x12, 0x6, 0x3, 0x55, 0x4, 0xa, + 0x13, 0xb, 0x47, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x20, 0x49, + 0x6e, 0x63, 0x2e, 0x31, 0xc, 0x30, 0xa, 0x6, 0x3, 0x55, 0x4, + 0xb, 0x13, 0x3, 0x45, 0x6e, 0x67, 0x31, 0xc, 0x30, 0xa, 0x6, + 0x3, 0x55, 0x4, 0x3, 0x13, 0x3, 0x61, 0x67, 0x6c, 0x31, 0x1d, + 0x30, 0x1b, 0x6, 0x9, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0xd, 0x1, + 0x9, 0x1, 0x16, 0xe, 0x61, 0x67, 0x6c, 0x40, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x30, 0x1e, 0x17, + 0xd, 0x30, 0x39, 0x30, 0x39, 0x30, 0x39, 0x32, 0x32, 0x30, + 0x35, 0x34, 0x33, 0x5a, 0x17, 0xd, 0x31, 0x30, 0x30, 0x39, + 0x30, 0x39, 0x32, 0x32, 0x30, 0x35, 0x34, 0x33, 0x5a, 0x30, + 0x6a, 0x31, 0xb, 0x30, 0x9, 0x6, 0x3, 0x55, 0x4, 0x6, 0x13, + 0x2, 0x41, 0x55, 0x31, 0x13, 0x30, 0x11, 0x6, 0x3, 0x55, 0x4, + 0x8, 0x13, 0xa, 0x53, 0x6f, 0x6d, 0x65, 0x2d, 0x53, 0x74, 0x61, + 0x74, 0x65, 0x31, 0x21, 0x30, 0x1f, 0x6, 0x3, 0x55, 0x4, 0xa, + 0x13, 0x18, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, + 0x20, 0x57, 0x69, 0x64, 0x67, 0x69, 0x74, 0x73, 0x20, 0x50, + 0x74, 0x79, 0x20, 0x4c, 0x74, 0x64, 0x31, 0x23, 0x30, 0x21, + 0x6, 0x3, 0x55, 0x4, 0x3, 0x13, 0x1a, 0x65, 0x75, 0x72, 0x6f, + 0x70, 0x61, 0x2e, 0x73, 0x66, 0x6f, 0x2e, 0x63, 0x6f, 0x72, + 0x70, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x63, + 0x6f, 0x6d, 0x30, 0x82, 0x2, 0x22, 0x30, 0xd, 0x6, 0x9, 0x2a, + 0x86, 0x48, 0x86, 0xf7, 0xd, 0x1, 0x1, 0x1, 0x5, 0x0, 0x3, + 0x82, 0x2, 0xf, 0x0, 0x30, 0x82, 0x2, 0xa, 0x2, 0x82, 0x2, 0x1, + 0x0, 0xba, 0xa6, 0x6, 0x2d, 0xef, 0xf1, 0x22, 0x6c, 0x10, 0xee, + 0x98, 0x4, 0x85, 0xf9, 0x2d, 0x2a, 0xbe, 0xa7, 0x4b, 0xfd, + 0xff, 0xa7, 0xa2, 0xe2, 0x70, 0xd9, 0x3d, 0x50, 0x73, 0xc1, + 0xa6, 0x5c, 0x5f, 0xe2, 0x89, 0x47, 0x2b, 0xb2, 0xc, 0xe1, + 0xfc, 0x57, 0x7c, 0xa4, 0x38, 0x73, 0xfd, 0x3f, 0x7a, 0x42, + 0xf, 0x48, 0xc9, 0x1e, 0x33, 0xb6, 0xe7, 0x36, 0x31, 0x7a, + 0xe6, 0x3e, 0xb6, 0xbe, 0xcc, 0xb, 0x92, 0x7f, 0x96, 0xde, + 0xbc, 0x54, 0x5e, 0x4a, 0xab, 0xe9, 0x22, 0xdf, 0x3, 0xba, + 0xfd, 0x7, 0x6c, 0x36, 0xdc, 0xaa, 0xcf, 0xd7, 0x28, 0xf7, + 0xc1, 0xb8, 0xc0, 0xde, 0xfa, 0x87, 0xfb, 0x6b, 0xd1, 0x82, + 0xcb, 0xb, 0xc0, 0x67, 0x86, 0x54, 0x25, 0xca, 0x74, 0xb6, + 0x1a, 0x83, 0xf5, 0xf2, 0x93, 0x4e, 0x35, 0x16, 0x38, 0x1c, + 0xc7, 0x63, 0x5b, 0x8d, 0x13, 0x84, 0x62, 0xb6, 0xb8, 0x66, + 0x24, 0x2a, 0xdb, 0x4, 0x65, 0xdb, 0x27, 0x6e, 0x3d, 0x6e, + 0xf8, 0x26, 0x6f, 0xb8, 0xe0, 0x74, 0x89, 0xc7, 0xd8, 0xf5, + 0x3b, 0x3d, 0x3e, 0x79, 0x86, 0xe8, 0xef, 0x3a, 0x82, 0x2c, + 0x41, 0x6d, 0xa8, 0xb1, 0x70, 0x45, 0x46, 0xea, 0xf8, 0xa4, + 0x5d, 0x70, 0x99, 0x58, 0x78, 0x5b, 0x89, 0x53, 0x2c, 0x3e, + 0xa9, 0x8d, 0xf2, 0xc, 0x51, 0xc3, 0xb, 0x20, 0x21, 0xc4, 0x5c, + 0x43, 0x7b, 0xdc, 0xf0, 0xe2, 0x73, 0xc2, 0x92, 0xdc, 0x4b, + 0x4d, 0x60, 0x2c, 0x28, 0x22, 0x57, 0x7e, 0xb6, 0x92, 0x1, + 0x8c, 0x4d, 0x50, 0x39, 0xbd, 0x4a, 0x3b, 0x2b, 0x89, 0x4, + 0xd5, 0xba, 0x12, 0xcc, 0x3d, 0x0, 0xb6, 0x67, 0xee, 0xcb, + 0x56, 0x75, 0x87, 0x4, 0xa8, 0xc, 0x2f, 0xbf, 0x6d, 0x1e, 0x80, + 0xdd, 0xad, 0x11, 0x7a, 0xa3, 0xed, 0xe, 0x3f, 0xd5, 0xc7, + 0x32, 0x3d, 0x97, 0x37, 0x3d, 0x8, 0x2, 0xdc, 0x52, 0x7c, 0x5, + 0xed, 0xa6, 0xb6, 0x59, 0x4c, 0xfb, 0xcd, 0x2a, 0x7f, 0x87, + 0x40, 0x69, 0xf, 0xdf, 0x5a, 0xf0, 0xad, 0x3a, 0xb4, 0x39, + 0xa6, 0x77, 0xde, 0xf6, 0x3b, 0x90, 0xbc, 0x2a, 0x3c, 0x63, + 0x45, 0x46, 0x94, 0x57, 0x2b, 0x4b, 0x2b, 0x3f, 0x33, 0x34, + 0xa0, 0xf4, 0xb2, 0x7f, 0xdb, 0x88, 0xd9, 0x56, 0xc, 0xd0, + 0xbd, 0xe7, 0x7f, 0x42, 0xea, 0x6c, 0xa7, 0xcf, 0x70, 0x56, + 0x27, 0x70, 0x23, 0x38, 0xed, 0xf9, 0x67, 0x52, 0x7f, 0x25, + 0x9f, 0x1f, 0x4f, 0xef, 0xd5, 0xa4, 0xb, 0xbe, 0xee, 0x5c, + 0x31, 0x8b, 0x96, 0xff, 0xf0, 0xb4, 0x5b, 0x69, 0xe5, 0x63, + 0x38, 0x8b, 0xe5, 0xb9, 0x44, 0x4e, 0x7b, 0x4c, 0xf0, 0xcc, + 0xb6, 0xc9, 0xb3, 0x40, 0x47, 0x2d, 0xc8, 0x8, 0x7c, 0x42, 0x6, + 0xfd, 0x91, 0x36, 0xd3, 0xc8, 0x59, 0x5, 0x66, 0xf8, 0x4f, + 0x7f, 0x5b, 0x9a, 0xf0, 0x87, 0x62, 0x9, 0x47, 0x80, 0xc7, + 0x9f, 0x62, 0xc7, 0xc1, 0x45, 0x66, 0x4f, 0xf8, 0x47, 0x35, + 0x37, 0x41, 0x2e, 0x5e, 0x54, 0x87, 0xa, 0xf8, 0x59, 0xce, + 0xc9, 0x84, 0x4e, 0xc0, 0x56, 0xa6, 0xf5, 0xe2, 0x95, 0x4d, + 0xcf, 0x59, 0x6b, 0xe6, 0x3c, 0x6c, 0xa7, 0xff, 0xe7, 0x70, + 0x8c, 0x1f, 0xb5, 0xbe, 0xe6, 0x65, 0xd, 0xc3, 0x17, 0x4d, + 0xd8, 0x9d, 0xaf, 0xa3, 0x26, 0x1a, 0x73, 0xc7, 0xc0, 0x90, + 0xa1, 0xe2, 0xd3, 0xe9, 0xa4, 0x2, 0x1f, 0x2c, 0xe6, 0x1f, + 0xaf, 0xc, 0xac, 0x71, 0xd3, 0xe0, 0xc0, 0xb, 0x3f, 0x25, 0x2c, + 0x3e, 0x89, 0xa3, 0x9f, 0xbd, 0x46, 0x35, 0x3b, 0x43, 0x79, + 0x60, 0x17, 0x89, 0xb8, 0xc2, 0x81, 0xd2, 0xfa, 0x88, 0x70, + 0x2, 0x8, 0x5e, 0x87, 0xb5, 0xb9, 0x74, 0xcc, 0x69, 0x46, 0xd0, + 0xe0, 0xa, 0xf9, 0xb7, 0x67, 0xd1, 0x2, 0x3, 0x1, 0x0, 0x1, + 0x30, 0xd, 0x6, 0x9, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0xd, 0x1, + 0x1, 0x5, 0x5, 0x0, 0x3, 0x81, 0x81, 0x0, 0xe2, 0xa7, 0x40, + 0x3b, 0x6a, 0x5, 0x5d, 0xb8, 0xbd, 0x5a, 0xaf, 0x7a, 0xc2, + 0xf9, 0x76, 0x79, 0xb2, 0x16, 0x91, 0x4f, 0x5c, 0x19, 0xf7, + 0x59, 0xd1, 0x5f, 0xc0, 0x3e, 0x83, 0xcc, 0x46, 0xf6, 0xd5, + 0x45, 0x46, 0xe6, 0x4a, 0xe6, 0x26, 0x2b, 0xbe, 0x56, 0x28, + 0xa9, 0x39, 0x12, 0x4f, 0x18, 0xc9, 0x20, 0xa0, 0xbe, 0x81, + 0x8d, 0x4d, 0xb5, 0x5c, 0x8f, 0x79, 0x55, 0x6, 0xee, 0x9f, + 0x93, 0xfc, 0xbe, 0xf8, 0x3c, 0x67, 0xe1, 0x67, 0x16, 0xcb, + 0xd1, 0x15, 0x8d, 0xde, 0xd4, 0x20, 0xb3, 0x1, 0x54, 0x18, + 0x66, 0xc5, 0x24, 0x2f, 0xd, 0x88, 0xef, 0x32, 0x3f, 0x74, + 0xd9, 0x56, 0x74, 0x1f, 0x17, 0xa7, 0xbb, 0xfe, 0xdf, 0xf, + 0x6c, 0x5f, 0x93, 0x77, 0x27, 0xf5, 0xae, 0x27, 0x52, 0x8e, + 0x3b, 0x99, 0xb6, 0xea, 0x44, 0x65, 0x1d, 0xa, 0x3b, 0x13, + 0x34, 0xf7, 0xf7, 0xbe, 0x20, + }, +} + +var privateKey = &Block{Type: "RSA PRIVATE KEY", + Headers: map[string]string{"DEK-Info": "DES-EDE3-CBC,80C7C7A09690757A", "Proc-Type": "4,ENCRYPTED"}, + Bytes: []uint8{0x79, 0xa, 0x79, 0x66, 0x41, 0xfa, 0xb, + 0x21, 0xc1, 0xcf, 0xb0, 0x59, 0x7d, 0x43, 0xf1, 0xc8, 0xb0, + 0x82, 0x99, 0xfb, 0x6c, 0x4, 0x9e, 0xc7, 0x96, 0xa1, 0x9b, + 0xf0, 0xb7, 0x76, 0xd5, 0xc4, 0xb0, 0x9f, 0x35, 0x99, 0xe3, + 0xf4, 0x88, 0x96, 0x1c, 0xab, 0x52, 0xdb, 0x1f, 0xc3, 0xb5, + 0x51, 0xd9, 0x34, 0xf0, 0x3, 0xea, 0x1d, 0xa3, 0xd7, 0xb1, + 0xec, 0x67, 0x71, 0x39, 0x36, 0x87, 0xf2, 0x86, 0x45, 0xba, + 0x62, 0x11, 0xa2, 0x21, 0x23, 0x1e, 0xc9, 0x3c, 0x53, 0xb0, + 0x61, 0x9e, 0xda, 0x7e, 0x7a, 0x49, 0xf, 0x3f, 0xbf, 0x71, + 0xba, 0x79, 0xcd, 0xee, 0x16, 0xfb, 0x86, 0x48, 0x6b, 0xc8, + 0x60, 0xd0, 0x66, 0x0, 0x3b, 0xb3, 0x67, 0x10, 0x1d, 0xe, + 0xf5, 0xbf, 0x78, 0x30, 0x4f, 0x60, 0x40, 0x8, 0x7, 0xed, + 0xdb, 0xa8, 0xc2, 0x8d, 0xb3, 0x35, 0xc2, 0x3, 0x73, 0x6, + 0x72, 0x7c, 0x33, 0x44, 0x73, 0x9f, 0xaf, 0x18, 0x5a, 0xa6, + 0x8f, 0xb5, 0xd0, 0x6e, 0xf2, 0xa6, 0xff, 0xcd, 0x54, 0x79, + 0x24, 0x1d, 0x66, 0x9e, 0xab, 0xd3, 0x49, 0xb1, 0x6c, 0x7c, + 0x5e, 0x72, 0x31, 0xce, 0xa8, 0xd9, 0x64, 0x3d, 0x8c, 0x90, + 0xa5, 0xe8, 0xac, 0xa9, 0x9c, 0xc9, 0x7e, 0x6, 0x92, 0xfe, + 0x76, 0x5b, 0xdd, 0x8d, 0x85, 0x4a, 0xe2, 0x5f, 0x65, 0x62, + 0xc0, 0x75, 0x1e, 0xcd, 0xdd, 0xfb, 0x30, 0xc1, 0x6d, 0x6c, + 0x68, 0xb3, 0xcd, 0x7c, 0xcf, 0x38, 0x52, 0xf3, 0xfc, 0xba, + 0x78, 0x87, 0xe2, 0x48, 0x6d, 0xbb, 0x72, 0xaa, 0xdb, 0x85, + 0x44, 0x5b, 0x9a, 0x8, 0x92, 0x7c, 0x35, 0xe3, 0x9d, 0xc2, + 0xb8, 0x9b, 0xfa, 0x4b, 0x71, 0xe, 0xe6, 0x98, 0x12, 0xfa, + 0x98, 0x53, 0xab, 0xdc, 0xf8, 0xf8, 0x8d, 0xea, 0x73, 0x80, + 0xe8, 0x85, 0x44, 0xbe, 0x8c, 0x28, 0x4f, 0xff, 0x87, 0xad, + 0x9f, 0x66, 0xf2, 0x35, 0x12, 0xb6, 0x2b, 0xf1, 0x5b, 0xa7, + 0x1c, 0x4c, 0xa4, 0x80, 0xa3, 0x61, 0xa, 0x10, 0xef, 0x6f, + 0x54, 0x4e, 0xe8, 0xb1, 0x1f, 0xa8, 0xa0, 0x6e, 0xf8, 0x8b, + 0xf0, 0x4c, 0x1e, 0x9f, 0x6, 0x15, 0xb7, 0x4c, 0x22, 0xcc, + 0x9d, 0x53, 0xd8, 0x56, 0x86, 0x30, 0x3, 0xe7, 0xc7, 0x35, + 0x39, 0x99, 0x66, 0x6f, 0x1, 0x12, 0x89, 0xe5, 0x20, 0xc7, + 0xc8, 0xf8, 0xa9, 0x2d, 0x8d, 0x65, 0x20, 0xe0, 0xe8, 0x9a, + 0xf8, 0xd2, 0x3a, 0xbc, 0xa7, 0xf, 0xa4, 0x5a, 0x31, 0x85, + 0x29, 0xe5, 0x1a, 0x0, 0xad, 0xbd, 0xf1, 0x1a, 0xab, 0x9f, + 0xc7, 0xe2, 0x41, 0xc9, 0xa7, 0xde, 0x5e, 0x3a, 0x99, 0xc5, + 0xb0, 0xbb, 0xe2, 0xc3, 0x6b, 0x1e, 0x1c, 0xd9, 0x33, 0xfc, + 0xbc, 0x49, 0x31, 0x13, 0xf5, 0x5f, 0x8d, 0xbc, 0xf8, 0x3a, + 0x58, 0x5b, 0xf4, 0x3, 0xff, 0x42, 0x54, 0xb4, 0x94, 0xae, + 0xb, 0xfa, 0xbd, 0x15, 0x41, 0xcb, 0xe9, 0x12, 0x37, 0xbb, + 0xf0, 0x2c, 0xae, 0xff, 0x29, 0x9, 0xbe, 0x3e, 0xf, 0xb8, + 0xb5, 0x8d, 0x6f, 0xc0, 0x30, 0x87, 0x49, 0x4e, 0xdc, 0x9c, + 0x38, 0x6b, 0x1b, 0x3f, 0x14, 0x0, 0xef, 0x21, 0xcc, 0x5a, + 0xe8, 0xc7, 0x99, 0xe6, 0xf5, 0x5, 0x38, 0x87, 0x70, 0xbd, + 0xec, 0xdc, 0xaa, 0xc9, 0xbb, 0x5b, 0xc3, 0x56, 0xbb, 0x21, + 0xbf, 0x17, 0xe7, 0xcf, 0xbd, 0x4a, 0x4d, 0xe3, 0x67, 0x64, + 0x7f, 0x14, 0x6c, 0xea, 0x93, 0xe6, 0x2e, 0xdf, 0x3c, 0x6c, + 0x97, 0xb5, 0x8d, 0x57, 0x25, 0xd7, 0x5c, 0x6c, 0x8f, 0xd0, + 0xea, 0xd5, 0xdf, 0xc0, 0x71, 0x6c, 0x65, 0xcd, 0xb4, 0x4f, + 0x34, 0x9d, 0xb1, 0x52, 0xc1, 0xb7, 0x9, 0x2c, 0x51, 0xa7, + 0x29, 0x6c, 0x3a, 0x20, 0x70, 0x45, 0x4c, 0x72, 0xd9, 0xcd, + 0xac, 0x1f, 0x5, 0x22, 0xb0, 0x77, 0xbd, 0x91, 0x2f, 0xf5, + 0xd, 0x19, 0xd5, 0x57, 0x98, 0xee, 0x79, 0x93, 0xa4, 0x5f, + 0xba, 0x6b, 0xec, 0xf6, 0x3f, 0xb6, 0x9c, 0x2f, 0xb8, 0xfa, + 0x3, 0xa3, 0xeb, 0xdf, 0xea, 0xbc, 0x3f, 0xd0, 0x8f, 0x88, + 0xf0, 0xbb, 0xcc, 0x5a, 0x27, 0x57, 0x3b, 0x95, 0x3f, 0x20, + 0x7c, 0x19, 0xc8, 0x46, 0x47, 0x68, 0x72, 0xb7, 0x28, 0x8e, + 0x56, 0x9b, 0x83, 0xf7, 0xe9, 0x3c, 0x85, 0xcb, 0xb0, 0x65, + 0x60, 0x1a, 0x52, 0x85, 0x6d, 0x58, 0x84, 0x39, 0xd9, 0xa2, + 0x92, 0xd2, 0x9d, 0x7d, 0x1b, 0xdf, 0x61, 0x85, 0xbf, 0x88, + 0x54, 0x3, 0x42, 0xe1, 0xa9, 0x24, 0x74, 0x75, 0x78, 0x48, + 0xff, 0x22, 0xec, 0xc5, 0x4d, 0x66, 0x17, 0xd4, 0x9a, + }, +} + +var privateKey2 = &Block{Type: "RSA PRIVATE KEY", + Headers: map[string]string{}, + Bytes: []uint8{0x30, 0x82, 0x1, 0x3a, 0x2, 0x1, 0x0, 0x2, + 0x41, 0x0, 0xb2, 0x99, 0xf, 0x49, 0xc4, 0x7d, 0xfa, 0x8c, + 0xd4, 0x0, 0xae, 0x6a, 0x4d, 0x1b, 0x8a, 0x3b, 0x6a, 0x13, + 0x64, 0x2b, 0x23, 0xf2, 0x8b, 0x0, 0x3b, 0xfb, 0x97, 0x79, + 0xa, 0xde, 0x9a, 0x4c, 0xc8, 0x2b, 0x8b, 0x2a, 0x81, 0x74, + 0x7d, 0xde, 0xc0, 0x8b, 0x62, 0x96, 0xe5, 0x3a, 0x8, 0xc3, + 0x31, 0x68, 0x7e, 0xf2, 0x5c, 0x4b, 0xf4, 0x93, 0x6b, 0xa1, + 0xc0, 0xe6, 0x4, 0x1e, 0x9d, 0x15, 0x2, 0x3, 0x1, 0x0, 0x1, + 0x2, 0x41, 0x0, 0x8a, 0xbd, 0x6a, 0x69, 0xf4, 0xd1, 0xa4, + 0xb4, 0x87, 0xf0, 0xab, 0x8d, 0x7a, 0xae, 0xfd, 0x38, 0x60, + 0x94, 0x5, 0xc9, 0x99, 0x98, 0x4e, 0x30, 0xf5, 0x67, 0xe1, + 0xe8, 0xae, 0xef, 0xf4, 0x4e, 0x8b, 0x18, 0xbd, 0xb1, 0xec, + 0x78, 0xdf, 0xa3, 0x1a, 0x55, 0xe3, 0x2a, 0x48, 0xd7, 0xfb, + 0x13, 0x1f, 0x5a, 0xf1, 0xf4, 0x4d, 0x7d, 0x6b, 0x2c, 0xed, + 0x2a, 0x9d, 0xf5, 0xe5, 0xae, 0x45, 0x35, 0x2, 0x21, 0x0, + 0xda, 0xb2, 0xf1, 0x80, 0x48, 0xba, 0xa6, 0x8d, 0xe7, 0xdf, + 0x4, 0xd2, 0xd3, 0x5d, 0x5d, 0x80, 0xe6, 0xe, 0x2d, 0xfa, + 0x42, 0xd5, 0xa, 0x9b, 0x4, 0x21, 0x90, 0x32, 0x71, 0x5e, + 0x46, 0xb3, 0x2, 0x21, 0x0, 0xd1, 0xf, 0x2e, 0x66, 0xb1, + 0xd0, 0xc1, 0x3f, 0x10, 0xef, 0x99, 0x27, 0xbf, 0x53, 0x24, + 0xa3, 0x79, 0xca, 0x21, 0x81, 0x46, 0xcb, 0xf9, 0xca, 0xfc, + 0x79, 0x52, 0x21, 0xf1, 0x6a, 0x31, 0x17, 0x2, 0x20, 0x21, + 0x2, 0x89, 0x79, 0x37, 0x81, 0x14, 0xca, 0xae, 0x88, 0xf7, + 0xd, 0x6b, 0x61, 0xd8, 0x4f, 0x30, 0x6a, 0x4b, 0x7e, 0x4e, + 0xc0, 0x21, 0x4d, 0xac, 0x9d, 0xf4, 0x49, 0xe8, 0xda, 0xb6, + 0x9, 0x2, 0x20, 0x16, 0xb3, 0xec, 0x59, 0x10, 0xa4, 0x57, + 0xe8, 0xe, 0x61, 0xc6, 0xa3, 0xf, 0x5e, 0xeb, 0x12, 0xa9, + 0xae, 0x2e, 0xb7, 0x48, 0x45, 0xec, 0x69, 0x83, 0xc3, 0x75, + 0xc, 0xe4, 0x97, 0xa0, 0x9f, 0x2, 0x20, 0x69, 0x52, 0xb4, + 0x6, 0xe8, 0x50, 0x60, 0x71, 0x4c, 0x3a, 0xb7, 0x66, 0xba, + 0xd, 0x8a, 0xc9, 0xb7, 0xd, 0xa3, 0x8, 0x6c, 0xa3, 0xf2, + 0x62, 0xb0, 0x2a, 0x84, 0xaa, 0x2f, 0xd6, 0x1e, 0x55, + }, +} + +var pemPrivateKey = `-----BEGIN RSA PRIVATE KEY----- +MIIBOgIBAAJBALKZD0nEffqM1ACuak0bijtqE2QrI/KLADv7l3kK3ppMyCuLKoF0 +fd7Ai2KW5ToIwzFofvJcS/STa6HA5gQenRUCAwEAAQJBAIq9amn00aS0h/CrjXqu +/ThglAXJmZhOMPVn4eiu7/ROixi9sex436MaVeMqSNf7Ex9a8fRNfWss7Sqd9eWu +RTUCIQDasvGASLqmjeffBNLTXV2A5g4t+kLVCpsEIZAycV5GswIhANEPLmax0ME/ +EO+ZJ79TJKN5yiGBRsv5yvx5UiHxajEXAiAhAol5N4EUyq6I9w1rYdhPMGpLfk7A +IU2snfRJ6Nq2CQIgFrPsWRCkV+gOYcajD17rEqmuLrdIRexpg8N1DOSXoJ8CIGlS +tAboUGBxTDq3ZroNism3DaMIbKPyYrAqhKov1h5V +-----END RSA PRIVATE KEY----- +` diff --git a/libgo/go/exec/exec.go b/libgo/go/exec/exec.go new file mode 100644 index 000000000..ba9bd2472 --- /dev/null +++ b/libgo/go/exec/exec.go @@ -0,0 +1,183 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The exec package runs external commands. +package exec + +import ( + "os" +) + +// Arguments to Run. +const ( + DevNull = iota + PassThrough + Pipe + MergeWithStdout +) + +// A Cmd represents a running command. +// Stdin, Stdout, and Stderr are Files representing pipes +// connected to the running command's standard input, output, and error, +// or else nil, depending on the arguments to Run. +// Pid is the running command's operating system process ID. +type Cmd struct { + Stdin *os.File + Stdout *os.File + Stderr *os.File + Pid int +} + +// Given mode (DevNull, etc), return file for child +// and file to record in Cmd structure. +func modeToFiles(mode, fd int) (*os.File, *os.File, os.Error) { + switch mode { + case DevNull: + rw := os.O_WRONLY + if fd == 0 { + rw = os.O_RDONLY + } + f, err := os.Open(os.DevNull, rw, 0) + return f, nil, err + case PassThrough: + switch fd { + case 0: + return os.Stdin, nil, nil + case 1: + return os.Stdout, nil, nil + case 2: + return os.Stderr, nil, nil + } + case Pipe: + r, w, err := os.Pipe() + if err != nil { + return nil, nil, err + } + if fd == 0 { + return r, w, nil + } + return w, r, nil + } + return nil, nil, os.EINVAL +} + +// Run starts the named binary running with +// arguments argv and environment envv. +// It returns a pointer to a new Cmd representing +// the command or an error. +// +// The parameters stdin, stdout, and stderr +// specify how to handle standard input, output, and error. +// The choices are DevNull (connect to /dev/null), +// PassThrough (connect to the current process's standard stream), +// Pipe (connect to an operating system pipe), and +// MergeWithStdout (only for standard error; use the same +// file descriptor as was used for standard output). +// If a parameter is Pipe, then the corresponding field (Stdin, Stdout, Stderr) +// of the returned Cmd is the other end of the pipe. +// Otherwise the field in Cmd is nil. +func Run(name string, argv, envv []string, dir string, stdin, stdout, stderr int) (p *Cmd, err os.Error) { + p = new(Cmd) + var fd [3]*os.File + + if fd[0], p.Stdin, err = modeToFiles(stdin, 0); err != nil { + goto Error + } + if fd[1], p.Stdout, err = modeToFiles(stdout, 1); err != nil { + goto Error + } + if stderr == MergeWithStdout { + fd[2] = fd[1] + } else if fd[2], p.Stderr, err = modeToFiles(stderr, 2); err != nil { + goto Error + } + + // Run command. + p.Pid, err = os.ForkExec(name, argv, envv, dir, fd[0:]) + if err != nil { + goto Error + } + if fd[0] != os.Stdin { + fd[0].Close() + } + if fd[1] != os.Stdout { + fd[1].Close() + } + if fd[2] != os.Stderr && fd[2] != fd[1] { + fd[2].Close() + } + return p, nil + +Error: + if fd[0] != os.Stdin && fd[0] != nil { + fd[0].Close() + } + if fd[1] != os.Stdout && fd[1] != nil { + fd[1].Close() + } + if fd[2] != os.Stderr && fd[2] != nil && fd[2] != fd[1] { + fd[2].Close() + } + if p.Stdin != nil { + p.Stdin.Close() + } + if p.Stdout != nil { + p.Stdout.Close() + } + if p.Stderr != nil { + p.Stderr.Close() + } + return nil, err +} + +// Wait waits for the running command p, +// returning the Waitmsg returned by os.Wait and an error. +// The options are passed through to os.Wait. +// Setting options to 0 waits for p to exit; +// other options cause Wait to return for other +// process events; see package os for details. +func (p *Cmd) Wait(options int) (*os.Waitmsg, os.Error) { + if p.Pid <= 0 { + return nil, os.ErrorString("exec: invalid use of Cmd.Wait") + } + w, err := os.Wait(p.Pid, options) + if w != nil && (w.Exited() || w.Signaled()) { + p.Pid = -1 + } + return w, err +} + +// Close waits for the running command p to exit, +// if it hasn't already, and then closes the non-nil file descriptors +// p.Stdin, p.Stdout, and p.Stderr. +func (p *Cmd) Close() os.Error { + if p.Pid > 0 { + // Loop on interrupt, but + // ignore other errors -- maybe + // caller has already waited for pid. + _, err := p.Wait(0) + for err == os.EINTR { + _, err = p.Wait(0) + } + } + + // Close the FDs that are still open. + var err os.Error + if p.Stdin != nil && p.Stdin.Fd() >= 0 { + if err1 := p.Stdin.Close(); err1 != nil { + err = err1 + } + } + if p.Stdout != nil && p.Stdout.Fd() >= 0 { + if err1 := p.Stdout.Close(); err1 != nil && err != nil { + err = err1 + } + } + if p.Stderr != nil && p.Stderr != p.Stdout && p.Stderr.Fd() >= 0 { + if err1 := p.Stderr.Close(); err1 != nil && err != nil { + err = err1 + } + } + return err +} diff --git a/libgo/go/exec/exec_test.go b/libgo/go/exec/exec_test.go new file mode 100644 index 000000000..3a3d3b1a5 --- /dev/null +++ b/libgo/go/exec/exec_test.go @@ -0,0 +1,120 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package exec + +import ( + "io" + "io/ioutil" + "testing" + "os" + "runtime" +) + +func run(argv []string, stdin, stdout, stderr int) (p *Cmd, err os.Error) { + if runtime.GOOS == "windows" { + argv = append([]string{"cmd", "/c"}, argv...) + } + exe, err := LookPath(argv[0]) + if err != nil { + return nil, err + } + p, err = Run(exe, argv, nil, "", stdin, stdout, stderr) + return p, err +} + +func TestRunCat(t *testing.T) { + cmd, err := run([]string{"cat"}, Pipe, Pipe, DevNull) + if err != nil { + t.Fatal("run:", err) + } + io.WriteString(cmd.Stdin, "hello, world\n") + cmd.Stdin.Close() + buf, err := ioutil.ReadAll(cmd.Stdout) + if err != nil { + t.Fatal("read:", err) + } + if string(buf) != "hello, world\n" { + t.Fatalf("read: got %q", buf) + } + if err = cmd.Close(); err != nil { + t.Fatal("close:", err) + } +} + +func TestRunEcho(t *testing.T) { + cmd, err := run([]string{"sh", "-c", "echo hello world"}, + DevNull, Pipe, DevNull) + if err != nil { + t.Fatal("run:", err) + } + buf, err := ioutil.ReadAll(cmd.Stdout) + if err != nil { + t.Fatal("read:", err) + } + if string(buf) != "hello world\n" { + t.Fatalf("read: got %q", buf) + } + if err = cmd.Close(); err != nil { + t.Fatal("close:", err) + } +} + +func TestStderr(t *testing.T) { + cmd, err := run([]string{"sh", "-c", "echo hello world 1>&2"}, + DevNull, DevNull, Pipe) + if err != nil { + t.Fatal("run:", err) + } + buf, err := ioutil.ReadAll(cmd.Stderr) + if err != nil { + t.Fatal("read:", err) + } + if string(buf) != "hello world\n" { + t.Fatalf("read: got %q", buf) + } + if err = cmd.Close(); err != nil { + t.Fatal("close:", err) + } +} + +func TestMergeWithStdout(t *testing.T) { + cmd, err := run([]string{"sh", "-c", "echo hello world 1>&2"}, + DevNull, Pipe, MergeWithStdout) + if err != nil { + t.Fatal("run:", err) + } + buf, err := ioutil.ReadAll(cmd.Stdout) + if err != nil { + t.Fatal("read:", err) + } + if string(buf) != "hello world\n" { + t.Fatalf("read: got %q", buf) + } + if err = cmd.Close(); err != nil { + t.Fatal("close:", err) + } +} + +func TestAddEnvVar(t *testing.T) { + err := os.Setenv("NEWVAR", "hello world") + if err != nil { + t.Fatal("setenv:", err) + } + cmd, err := run([]string{"sh", "-c", "echo $NEWVAR"}, + DevNull, Pipe, DevNull) + if err != nil { + t.Fatal("run:", err) + } + buf, err := ioutil.ReadAll(cmd.Stdout) + if err != nil { + t.Fatal("read:", err) + } + if string(buf) != "hello world\n" { + t.Fatalf("read: got %q", buf) + } + if err = cmd.Close(); err != nil { + t.Fatal("close:", err) + } +} diff --git a/libgo/go/exec/lp_unix.go b/libgo/go/exec/lp_unix.go new file mode 100644 index 000000000..292e24fcc --- /dev/null +++ b/libgo/go/exec/lp_unix.go @@ -0,0 +1,45 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package exec + +import ( + "os" + "strings" +) + +func canExec(file string) bool { + d, err := os.Stat(file) + if err != nil { + return false + } + return d.IsRegular() && d.Permission()&0111 != 0 +} + +// LookPath searches for an executable binary named file +// in the directories named by the PATH environment variable. +// If file contains a slash, it is tried directly and the PATH is not consulted. +func LookPath(file string) (string, os.Error) { + // NOTE(rsc): I wish we could use the Plan 9 behavior here + // (only bypass the path if file begins with / or ./ or ../) + // but that would not match all the Unix shells. + + if strings.Contains(file, "/") { + if canExec(file) { + return file, nil + } + return "", &os.PathError{"lookpath", file, os.ENOENT} + } + pathenv := os.Getenv("PATH") + for _, dir := range strings.Split(pathenv, ":", -1) { + if dir == "" { + // Unix shell semantics: path element "" means "." + dir = "." + } + if canExec(dir + "/" + file) { + return dir + "/" + file, nil + } + } + return "", &os.PathError{"lookpath", file, os.ENOENT} +} diff --git a/libgo/go/exec/lp_windows.go b/libgo/go/exec/lp_windows.go new file mode 100644 index 000000000..7b56afa85 --- /dev/null +++ b/libgo/go/exec/lp_windows.go @@ -0,0 +1,66 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package exec + +import ( + "os" + "strings" +) + +func chkStat(file string) bool { + d, err := os.Stat(file) + if err != nil { + return false + } + return d.IsRegular() +} + +func canExec(file string, exts []string) (string, bool) { + if len(exts) == 0 { + return file, chkStat(file) + } + f := strings.ToLower(file) + for _, e := range exts { + if strings.HasSuffix(f, e) { + return file, chkStat(file) + } + } + for _, e := range exts { + if f := file + e; chkStat(f) { + return f, true + } + } + return ``, false +} + +func LookPath(file string) (string, os.Error) { + exts := []string{} + if x := os.Getenv(`PATHEXT`); x != `` { + exts = strings.Split(strings.ToLower(x), `;`, -1) + for i, e := range exts { + if e == `` || e[0] != '.' { + exts[i] = `.` + e + } + } + } + if strings.Contains(file, `\`) || strings.Contains(file, `/`) { + if f, ok := canExec(file, exts); ok { + return f, nil + } + return ``, &os.PathError{"lookpath", file, os.ENOENT} + } + if pathenv := os.Getenv(`PATH`); pathenv == `` { + if f, ok := canExec(`.\`+file, exts); ok { + return f, nil + } + } else { + for _, dir := range strings.Split(pathenv, `;`, -1) { + if f, ok := canExec(dir+`\`+file, exts); ok { + return f, nil + } + } + } + return ``, &os.PathError{"lookpath", file, os.ENOENT} +} diff --git a/libgo/go/exp/README b/libgo/go/exp/README new file mode 100644 index 000000000..e602e3ac9 --- /dev/null +++ b/libgo/go/exp/README @@ -0,0 +1,3 @@ +This directory tree contains experimental packages and +unfinished code that is subject to even more change than the +rest of the Go tree. diff --git a/libgo/go/exp/datafmt/datafmt.go b/libgo/go/exp/datafmt/datafmt.go new file mode 100644 index 000000000..46c412342 --- /dev/null +++ b/libgo/go/exp/datafmt/datafmt.go @@ -0,0 +1,731 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* The datafmt package implements syntax-directed, type-driven formatting + of arbitrary data structures. Formatting a data structure consists of + two phases: first, a parser reads a format specification and builds a + "compiled" format. Then, the format can be applied repeatedly to + arbitrary values. Applying a format to a value evaluates to a []byte + containing the formatted value bytes, or nil. + + A format specification is a set of package declarations and format rules: + + Format = [ Entry { ";" Entry } [ ";" ] ] . + Entry = PackageDecl | FormatRule . + + (The syntax of a format specification is presented in the same EBNF + notation as used in the Go language specification. The syntax of white + space, comments, identifiers, and string literals is the same as in Go.) + + A package declaration binds a package name (such as 'ast') to a + package import path (such as '"go/ast"'). Each package used (in + a type name, see below) must be declared once before use. + + PackageDecl = PackageName ImportPath . + PackageName = identifier . + ImportPath = string . + + A format rule binds a rule name to a format expression. A rule name + may be a type name or one of the special names 'default' or '/'. + A type name may be the name of a predeclared type (for example, 'int', + 'float32', etc.), the package-qualified name of a user-defined type + (for example, 'ast.MapType'), or an identifier indicating the structure + of unnamed composite types ('array', 'chan', 'func', 'interface', 'map', + or 'ptr'). Each rule must have a unique name; rules can be declared in + any order. + + FormatRule = RuleName "=" Expression . + RuleName = TypeName | "default" | "/" . + TypeName = [ PackageName "." ] identifier . + + To format a value, the value's type name is used to select the format rule + (there is an override mechanism, see below). The format expression of the + selected rule specifies how the value is formatted. Each format expression, + when applied to a value, evaluates to a byte sequence or nil. + + In its most general form, a format expression is a list of alternatives, + each of which is a sequence of operands: + + Expression = [ Sequence ] { "|" [ Sequence ] } . + Sequence = Operand { Operand } . + + The formatted result produced by an expression is the result of the first + alternative sequence that evaluates to a non-nil result; if there is no + such alternative, the expression evaluates to nil. The result produced by + an operand sequence is the concatenation of the results of its operands. + If any operand in the sequence evaluates to nil, the entire sequence + evaluates to nil. + + There are five kinds of operands: + + Operand = Literal | Field | Group | Option | Repetition . + + Literals evaluate to themselves, with two substitutions. First, + %-formats expand in the manner of fmt.Printf, with the current value + passed as the parameter. Second, the current indentation (see below) + is inserted after every newline or form feed character. + + Literal = string . + + This table shows string literals applied to the value 42 and the + corresponding formatted result: + + "foo" foo + "%x" 2a + "x = %d" x = 42 + "%#x = %d" 0x2a = 42 + + A field operand is a field name optionally followed by an alternate + rule name. The field name may be an identifier or one of the special + names @ or *. + + Field = FieldName [ ":" RuleName ] . + FieldName = identifier | "@" | "*" . + + If the field name is an identifier, the current value must be a struct, + and there must be a field with that name in the struct. The same lookup + rules apply as in the Go language (for instance, the name of an anonymous + field is the unqualified type name). The field name denotes the field + value in the struct. If the field is not found, formatting is aborted + and an error message is returned. (TODO consider changing the semantics + such that if a field is not found, it evaluates to nil). + + The special name '@' denotes the current value. + + The meaning of the special name '*' depends on the type of the current + value: + + array, slice types array, slice element (inside {} only, see below) + interfaces value stored in interface + pointers value pointed to by pointer + + (Implementation restriction: channel, function and map types are not + supported due to missing reflection support). + + Fields are evaluated as follows: If the field value is nil, or an array + or slice element does not exist, the result is nil (see below for details + on array/slice elements). If the value is not nil the field value is + formatted (recursively) using the rule corresponding to its type name, + or the alternate rule name, if given. + + The following example shows a complete format specification for a + struct 'myPackage.Point'. Assume the package + + package myPackage // in directory myDir/myPackage + type Point struct { + name string; + x, y int; + } + + Applying the format specification + + myPackage "myDir/myPackage"; + int = "%d"; + hexInt = "0x%x"; + string = "---%s---"; + myPackage.Point = name "{" x ", " y:hexInt "}"; + + to the value myPackage.Point{"foo", 3, 15} results in + + ---foo---{3, 0xf} + + Finally, an operand may be a grouped, optional, or repeated expression. + A grouped expression ("group") groups a more complex expression (body) + so that it can be used in place of a single operand: + + Group = "(" [ Indentation ">>" ] Body ")" . + Indentation = Expression . + Body = Expression . + + A group body may be prefixed by an indentation expression followed by '>>'. + The indentation expression is applied to the current value like any other + expression and the result, if not nil, is appended to the current indentation + during the evaluation of the body (see also formatting state, below). + + An optional expression ("option") is enclosed in '[]' brackets. + + Option = "[" Body "]" . + + An option evaluates to its body, except that if the body evaluates to nil, + the option expression evaluates to an empty []byte. Thus an option's purpose + is to protect the expression containing the option from a nil operand. + + A repeated expression ("repetition") is enclosed in '{}' braces. + + Repetition = "{" Body [ "/" Separator ] "}" . + Separator = Expression . + + A repeated expression is evaluated as follows: The body is evaluated + repeatedly and its results are concatenated until the body evaluates + to nil. The result of the repetition is the (possibly empty) concatenation, + but it is never nil. An implicit index is supplied for the evaluation of + the body: that index is used to address elements of arrays or slices. If + the corresponding elements do not exist, the field denoting the element + evaluates to nil (which in turn may terminate the repetition). + + The body of a repetition may be followed by a '/' and a "separator" + expression. If the separator is present, it is invoked between repetitions + of the body. + + The following example shows a complete format specification for formatting + a slice of unnamed type. Applying the specification + + int = "%b"; + array = { * / ", " }; // array is the type name for an unnamed slice + + to the value '[]int{2, 3, 5, 7}' results in + + 10, 11, 101, 111 + + Default rule: If a format rule named 'default' is present, it is used for + formatting a value if no other rule was found. A common default rule is + + default = "%v" + + to provide default formatting for basic types without having to specify + a specific rule for each basic type. + + Global separator rule: If a format rule named '/' is present, it is + invoked with the current value between literals. If the separator + expression evaluates to nil, it is ignored. + + For instance, a global separator rule may be used to punctuate a sequence + of values with commas. The rules: + + default = "%v"; + / = ", "; + + will format an argument list by printing each one in its default format, + separated by a comma and a space. +*/ +package datafmt + +import ( + "bytes" + "fmt" + "go/token" + "io" + "os" + "reflect" + "runtime" +) + + +// ---------------------------------------------------------------------------- +// Format representation + +// Custom formatters implement the Formatter function type. +// A formatter is invoked with the current formatting state, the +// value to format, and the rule name under which the formatter +// was installed (the same formatter function may be installed +// under different names). The formatter may access the current state +// to guide formatting and use State.Write to append to the state's +// output. +// +// A formatter must return a boolean value indicating if it evaluated +// to a non-nil value (true), or a nil value (false). +// +type Formatter func(state *State, value interface{}, ruleName string) bool + + +// A FormatterMap is a set of custom formatters. +// It maps a rule name to a formatter function. +// +type FormatterMap map[string]Formatter + + +// A parsed format expression is built from the following nodes. +// +type ( + expr interface{} + + alternatives []expr // x | y | z + + sequence []expr // x y z + + literal [][]byte // a list of string segments, possibly starting with '%' + + field struct { + fieldName string // including "@", "*" + ruleName string // "" if no rule name specified + } + + group struct { + indent, body expr // (indent >> body) + } + + option struct { + body expr // [body] + } + + repetition struct { + body, separator expr // {body / separator} + } + + custom struct { + ruleName string + fun Formatter + } +) + + +// A Format is the result of parsing a format specification. +// The format may be applied repeatedly to format values. +// +type Format map[string]expr + + +// ---------------------------------------------------------------------------- +// Formatting + +// An application-specific environment may be provided to Format.Apply; +// the environment is available inside custom formatters via State.Env(). +// Environments must implement copying; the Copy method must return an +// complete copy of the receiver. This is necessary so that the formatter +// can save and restore an environment (in case of an absent expression). +// +// If the Environment doesn't change during formatting (this is under +// control of the custom formatters), the Copy function can simply return +// the receiver, and thus can be very light-weight. +// +type Environment interface { + Copy() Environment +} + + +// State represents the current formatting state. +// It is provided as argument to custom formatters. +// +type State struct { + fmt Format // format in use + env Environment // user-supplied environment + errors chan os.Error // not chan *Error (errors <- nil would be wrong!) + hasOutput bool // true after the first literal has been written + indent bytes.Buffer // current indentation + output bytes.Buffer // format output + linePos token.Position // position of line beginning (Column == 0) + default_ expr // possibly nil + separator expr // possibly nil +} + + +func newState(fmt Format, env Environment, errors chan os.Error) *State { + s := new(State) + s.fmt = fmt + s.env = env + s.errors = errors + s.linePos = token.Position{Line: 1} + + // if we have a default rule, cache it's expression for fast access + if x, found := fmt["default"]; found { + s.default_ = x + } + + // if we have a global separator rule, cache it's expression for fast access + if x, found := fmt["/"]; found { + s.separator = x + } + + return s +} + + +// Env returns the environment passed to Format.Apply. +func (s *State) Env() interface{} { return s.env } + + +// LinePos returns the position of the current line beginning +// in the state's output buffer. Line numbers start at 1. +// +func (s *State) LinePos() token.Position { return s.linePos } + + +// Pos returns the position of the next byte to be written to the +// output buffer. Line numbers start at 1. +// +func (s *State) Pos() token.Position { + offs := s.output.Len() + return token.Position{Line: s.linePos.Line, Column: offs - s.linePos.Offset, Offset: offs} +} + + +// Write writes data to the output buffer, inserting the indentation +// string after each newline or form feed character. It cannot return an error. +// +func (s *State) Write(data []byte) (int, os.Error) { + n := 0 + i0 := 0 + for i, ch := range data { + if ch == '\n' || ch == '\f' { + // write text segment and indentation + n1, _ := s.output.Write(data[i0 : i+1]) + n2, _ := s.output.Write(s.indent.Bytes()) + n += n1 + n2 + i0 = i + 1 + s.linePos.Offset = s.output.Len() + s.linePos.Line++ + } + } + n3, _ := s.output.Write(data[i0:]) + return n + n3, nil +} + + +type checkpoint struct { + env Environment + hasOutput bool + outputLen int + linePos token.Position +} + + +func (s *State) save() checkpoint { + saved := checkpoint{nil, s.hasOutput, s.output.Len(), s.linePos} + if s.env != nil { + saved.env = s.env.Copy() + } + return saved +} + + +func (s *State) restore(m checkpoint) { + s.env = m.env + s.output.Truncate(m.outputLen) +} + + +func (s *State) error(msg string) { + s.errors <- os.NewError(msg) + runtime.Goexit() +} + + +// TODO At the moment, unnamed types are simply mapped to the default +// names below. For instance, all unnamed arrays are mapped to +// 'array' which is not really sufficient. Eventually one may want +// to be able to specify rules for say an unnamed slice of T. +// + +func typename(typ reflect.Type) string { + switch typ.(type) { + case *reflect.ArrayType: + return "array" + case *reflect.SliceType: + return "array" + case *reflect.ChanType: + return "chan" + case *reflect.FuncType: + return "func" + case *reflect.InterfaceType: + return "interface" + case *reflect.MapType: + return "map" + case *reflect.PtrType: + return "ptr" + } + return typ.String() +} + +func (s *State) getFormat(name string) expr { + if fexpr, found := s.fmt[name]; found { + return fexpr + } + + if s.default_ != nil { + return s.default_ + } + + s.error(fmt.Sprintf("no format rule for type: '%s'", name)) + return nil +} + + +// eval applies a format expression fexpr to a value. If the expression +// evaluates internally to a non-nil []byte, that slice is appended to +// the state's output buffer and eval returns true. Otherwise, eval +// returns false and the state remains unchanged. +// +func (s *State) eval(fexpr expr, value reflect.Value, index int) bool { + // an empty format expression always evaluates + // to a non-nil (but empty) []byte + if fexpr == nil { + return true + } + + switch t := fexpr.(type) { + case alternatives: + // append the result of the first alternative that evaluates to + // a non-nil []byte to the state's output + mark := s.save() + for _, x := range t { + if s.eval(x, value, index) { + return true + } + s.restore(mark) + } + return false + + case sequence: + // append the result of all operands to the state's output + // unless a nil result is encountered + mark := s.save() + for _, x := range t { + if !s.eval(x, value, index) { + s.restore(mark) + return false + } + } + return true + + case literal: + // write separator, if any + if s.hasOutput { + // not the first literal + if s.separator != nil { + sep := s.separator // save current separator + s.separator = nil // and disable it (avoid recursion) + mark := s.save() + if !s.eval(sep, value, index) { + s.restore(mark) + } + s.separator = sep // enable it again + } + } + s.hasOutput = true + // write literal segments + for _, lit := range t { + if len(lit) > 1 && lit[0] == '%' { + // segment contains a %-format at the beginning + if lit[1] == '%' { + // "%%" is printed as a single "%" + s.Write(lit[1:]) + } else { + // use s instead of s.output to get indentation right + fmt.Fprintf(s, string(lit), value.Interface()) + } + } else { + // segment contains no %-formats + s.Write(lit) + } + } + return true // a literal never evaluates to nil + + case *field: + // determine field value + switch t.fieldName { + case "@": + // field value is current value + + case "*": + // indirection: operation is type-specific + switch v := value.(type) { + case *reflect.ArrayValue: + if v.Len() <= index { + return false + } + value = v.Elem(index) + + case *reflect.SliceValue: + if v.IsNil() || v.Len() <= index { + return false + } + value = v.Elem(index) + + case *reflect.MapValue: + s.error("reflection support for maps incomplete") + + case *reflect.PtrValue: + if v.IsNil() { + return false + } + value = v.Elem() + + case *reflect.InterfaceValue: + if v.IsNil() { + return false + } + value = v.Elem() + + case *reflect.ChanValue: + s.error("reflection support for chans incomplete") + + case *reflect.FuncValue: + s.error("reflection support for funcs incomplete") + + default: + s.error(fmt.Sprintf("error: * does not apply to `%s`", value.Type())) + } + + default: + // value is value of named field + var field reflect.Value + if sval, ok := value.(*reflect.StructValue); ok { + field = sval.FieldByName(t.fieldName) + if field == nil { + // TODO consider just returning false in this case + s.error(fmt.Sprintf("error: no field `%s` in `%s`", t.fieldName, value.Type())) + } + } + value = field + } + + // determine rule + ruleName := t.ruleName + if ruleName == "" { + // no alternate rule name, value type determines rule + ruleName = typename(value.Type()) + } + fexpr = s.getFormat(ruleName) + + mark := s.save() + if !s.eval(fexpr, value, index) { + s.restore(mark) + return false + } + return true + + case *group: + // remember current indentation + indentLen := s.indent.Len() + + // update current indentation + mark := s.save() + s.eval(t.indent, value, index) + // if the indentation evaluates to nil, the state's output buffer + // didn't change - either way it's ok to append the difference to + // the current identation + s.indent.Write(s.output.Bytes()[mark.outputLen:s.output.Len()]) + s.restore(mark) + + // format group body + mark = s.save() + b := true + if !s.eval(t.body, value, index) { + s.restore(mark) + b = false + } + + // reset indentation + s.indent.Truncate(indentLen) + return b + + case *option: + // evaluate the body and append the result to the state's output + // buffer unless the result is nil + mark := s.save() + if !s.eval(t.body, value, 0) { // TODO is 0 index correct? + s.restore(mark) + } + return true // an option never evaluates to nil + + case *repetition: + // evaluate the body and append the result to the state's output + // buffer until a result is nil + for i := 0; ; i++ { + mark := s.save() + // write separator, if any + if i > 0 && t.separator != nil { + // nil result from separator is ignored + mark := s.save() + if !s.eval(t.separator, value, i) { + s.restore(mark) + } + } + if !s.eval(t.body, value, i) { + s.restore(mark) + break + } + } + return true // a repetition never evaluates to nil + + case *custom: + // invoke the custom formatter to obtain the result + mark := s.save() + if !t.fun(s, value.Interface(), t.ruleName) { + s.restore(mark) + return false + } + return true + } + + panic("unreachable") + return false +} + + +// Eval formats each argument according to the format +// f and returns the resulting []byte and os.Error. If +// an error occurred, the []byte contains the partially +// formatted result. An environment env may be passed +// in which is available in custom formatters through +// the state parameter. +// +func (f Format) Eval(env Environment, args ...interface{}) ([]byte, os.Error) { + if f == nil { + return nil, os.NewError("format is nil") + } + + errors := make(chan os.Error) + s := newState(f, env, errors) + + go func() { + for _, v := range args { + fld := reflect.NewValue(v) + if fld == nil { + errors <- os.NewError("nil argument") + return + } + mark := s.save() + if !s.eval(s.getFormat(typename(fld.Type())), fld, 0) { // TODO is 0 index correct? + s.restore(mark) + } + } + errors <- nil // no errors + }() + + err := <-errors + return s.output.Bytes(), err +} + + +// ---------------------------------------------------------------------------- +// Convenience functions + +// Fprint formats each argument according to the format f +// and writes to w. The result is the total number of bytes +// written and an os.Error, if any. +// +func (f Format) Fprint(w io.Writer, env Environment, args ...interface{}) (int, os.Error) { + data, err := f.Eval(env, args...) + if err != nil { + // TODO should we print partial result in case of error? + return 0, err + } + return w.Write(data) +} + + +// Print formats each argument according to the format f +// and writes to standard output. The result is the total +// number of bytes written and an os.Error, if any. +// +func (f Format) Print(args ...interface{}) (int, os.Error) { + return f.Fprint(os.Stdout, nil, args...) +} + + +// Sprint formats each argument according to the format f +// and returns the resulting string. If an error occurs +// during formatting, the result string contains the +// partially formatted result followed by an error message. +// +func (f Format) Sprint(args ...interface{}) string { + var buf bytes.Buffer + _, err := f.Fprint(&buf, nil, args...) + if err != nil { + var i interface{} = args + fmt.Fprintf(&buf, "--- Sprint(%s) failed: %v", fmt.Sprint(i), err) + } + return buf.String() +} diff --git a/libgo/go/exp/datafmt/datafmt_test.go b/libgo/go/exp/datafmt/datafmt_test.go new file mode 100644 index 000000000..d7c70b21d --- /dev/null +++ b/libgo/go/exp/datafmt/datafmt_test.go @@ -0,0 +1,351 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package datafmt + +import ( + "fmt" + "testing" + "go/token" +) + + +var fset = token.NewFileSet() + + +func parse(t *testing.T, form string, fmap FormatterMap) Format { + f, err := Parse(fset, "", []byte(form), fmap) + if err != nil { + t.Errorf("Parse(%s): %v", form, err) + return nil + } + return f +} + + +func verify(t *testing.T, f Format, expected string, args ...interface{}) { + if f == nil { + return // allow other tests to run + } + result := f.Sprint(args...) + if result != expected { + t.Errorf( + "result : `%s`\nexpected: `%s`\n\n", + result, expected) + } +} + + +func formatter(s *State, value interface{}, rule_name string) bool { + switch rule_name { + case "/": + fmt.Fprintf(s, "%d %d %d", s.Pos().Line, s.LinePos().Column, s.Pos().Column) + return true + case "blank": + s.Write([]byte{' '}) + return true + case "int": + if value.(int)&1 == 0 { + fmt.Fprint(s, "even ") + } else { + fmt.Fprint(s, "odd ") + } + return true + case "nil": + return false + case "testing.T": + s.Write([]byte("testing.T")) + return true + } + panic("unreachable") + return false +} + + +func TestCustomFormatters(t *testing.T) { + fmap0 := FormatterMap{"/": formatter} + fmap1 := FormatterMap{"int": formatter, "blank": formatter, "nil": formatter} + fmap2 := FormatterMap{"testing.T": formatter} + + f := parse(t, `int=`, fmap0) + verify(t, f, ``, 1, 2, 3) + + f = parse(t, `int="#"`, nil) + verify(t, f, `###`, 1, 2, 3) + + f = parse(t, `int="#";string="%s"`, fmap0) + verify(t, f, "#1 0 1#1 0 7#1 0 13\n2 0 0foo2 0 8\n", 1, 2, 3, "\n", "foo", "\n") + + f = parse(t, ``, fmap1) + verify(t, f, `even odd even odd `, 0, 1, 2, 3) + + f = parse(t, `/ =@:blank; float64="#"`, fmap1) + verify(t, f, `# # #`, 0.0, 1.0, 2.0) + + f = parse(t, `float64=@:nil`, fmap1) + verify(t, f, ``, 0.0, 1.0, 2.0) + + f = parse(t, `testing "testing"; ptr=*`, fmap2) + verify(t, f, `testing.T`, t) + + // TODO needs more tests +} + + +// ---------------------------------------------------------------------------- +// Formatting of basic and simple composite types + +func check(t *testing.T, form, expected string, args ...interface{}) { + f := parse(t, form, nil) + if f == nil { + return // allow other tests to run + } + result := f.Sprint(args...) + if result != expected { + t.Errorf( + "format : %s\nresult : `%s`\nexpected: `%s`\n\n", + form, result, expected) + } +} + + +func TestBasicTypes(t *testing.T) { + check(t, ``, ``) + check(t, `bool=":%v"`, `:true:false`, true, false) + check(t, `int="%b %d %o 0x%x"`, `101010 42 52 0x2a`, 42) + + check(t, `int="%"`, `%`, 42) + check(t, `int="%%"`, `%`, 42) + check(t, `int="**%%**"`, `**%**`, 42) + check(t, `int="%%%%%%"`, `%%%`, 42) + check(t, `int="%%%d%%"`, `%42%`, 42) + + const i = -42 + const is = `-42` + check(t, `int ="%d"`, is, i) + check(t, `int8 ="%d"`, is, int8(i)) + check(t, `int16="%d"`, is, int16(i)) + check(t, `int32="%d"`, is, int32(i)) + check(t, `int64="%d"`, is, int64(i)) + + const u = 42 + const us = `42` + check(t, `uint ="%d"`, us, uint(u)) + check(t, `uint8 ="%d"`, us, uint8(u)) + check(t, `uint16="%d"`, us, uint16(u)) + check(t, `uint32="%d"`, us, uint32(u)) + check(t, `uint64="%d"`, us, uint64(u)) + + const f = 3.141592 + const fs = `3.141592` + check(t, `float64="%g"`, fs, f) + check(t, `float32="%g"`, fs, float32(f)) + check(t, `float64="%g"`, fs, float64(f)) +} + + +func TestArrayTypes(t *testing.T) { + var a0 [10]int + check(t, `array="array";`, `array`, a0) + + a1 := [...]int{1, 2, 3} + check(t, `array="array";`, `array`, a1) + check(t, `array={*}; int="%d";`, `123`, a1) + check(t, `array={* / ", "}; int="%d";`, `1, 2, 3`, a1) + check(t, `array={* / *}; int="%d";`, `12233`, a1) + + a2 := []interface{}{42, "foo", 3.14} + check(t, `array={* / ", "}; interface=*; string="bar"; default="%v";`, `42, bar, 3.14`, a2) +} + + +func TestChanTypes(t *testing.T) { + var c0 chan int + check(t, `chan="chan"`, `chan`, c0) + + c1 := make(chan int) + go func() { c1 <- 42 }() + check(t, `chan="chan"`, `chan`, c1) + // check(t, `chan=*`, `42`, c1); // reflection support for chans incomplete +} + + +func TestFuncTypes(t *testing.T) { + var f0 func() int + check(t, `func="func"`, `func`, f0) + + f1 := func() int { return 42 } + check(t, `func="func"`, `func`, f1) + // check(t, `func=*`, `42`, f1); // reflection support for funcs incomplete +} + + +func TestMapTypes(t *testing.T) { + var m0 map[string]int + check(t, `map="map"`, `map`, m0) + + m1 := map[string]int{} + check(t, `map="map"`, `map`, m1) + // check(t, `map=*`, ``, m1); // reflection support for maps incomplete +} + + +func TestPointerTypes(t *testing.T) { + var p0 *int + check(t, `ptr="ptr"`, `ptr`, p0) + check(t, `ptr=*`, ``, p0) + check(t, `ptr=*|"nil"`, `nil`, p0) + + x := 99991 + p1 := &x + check(t, `ptr="ptr"`, `ptr`, p1) + check(t, `ptr=*; int="%d"`, `99991`, p1) +} + + +func TestDefaultRule(t *testing.T) { + check(t, `default="%v"`, `42foo3.14`, 42, "foo", 3.14) + check(t, `default="%v"; int="%x"`, `abcdef`, 10, 11, 12, 13, 14, 15) + check(t, `default="%v"; int="%x"`, `ab**ef`, 10, 11, "**", 14, 15) + check(t, `default="%x"; int=@:default`, `abcdef`, 10, 11, 12, 13, 14, 15) +} + + +func TestGlobalSeparatorRule(t *testing.T) { + check(t, `int="%d"; / ="-"`, `1-2-3-4`, 1, 2, 3, 4) + check(t, `int="%x%x"; / ="*"`, `aa*aa`, 10, 10) +} + + +// ---------------------------------------------------------------------------- +// Formatting of a struct + +type T1 struct { + a int +} + +const F1 = `datafmt "datafmt";` + + `int = "%d";` + + `datafmt.T1 = "<" a ">";` + +func TestStruct1(t *testing.T) { check(t, F1, "<42>", T1{42}) } + + +// ---------------------------------------------------------------------------- +// Formatting of a struct with an optional field (ptr) + +type T2 struct { + s string + p *T1 +} + +const F2a = F1 + + `string = "%s";` + + `ptr = *;` + + `datafmt.T2 = s ["-" p "-"];` + +const F2b = F1 + + `string = "%s";` + + `ptr = *;` + + `datafmt.T2 = s ("-" p "-" | "empty");` + +func TestStruct2(t *testing.T) { + check(t, F2a, "foo", T2{"foo", nil}) + check(t, F2a, "bar-<17>-", T2{"bar", &T1{17}}) + check(t, F2b, "fooempty", T2{"foo", nil}) +} + + +// ---------------------------------------------------------------------------- +// Formatting of a struct with a repetitive field (slice) + +type T3 struct { + s string + a []int +} + +const F3a = `datafmt "datafmt";` + + `default = "%v";` + + `array = *;` + + `datafmt.T3 = s {" " a a / ","};` + +const F3b = `datafmt "datafmt";` + + `int = "%d";` + + `string = "%s";` + + `array = *;` + + `nil = ;` + + `empty = *:nil;` + + `datafmt.T3 = s [a:empty ": " {a / "-"}]` + +func TestStruct3(t *testing.T) { + check(t, F3a, "foo", T3{"foo", nil}) + check(t, F3a, "foo 00, 11, 22", T3{"foo", []int{0, 1, 2}}) + check(t, F3b, "bar", T3{"bar", nil}) + check(t, F3b, "bal: 2-3-5", T3{"bal", []int{2, 3, 5}}) +} + + +// ---------------------------------------------------------------------------- +// Formatting of a struct with alternative field + +type T4 struct { + x *int + a []int +} + +const F4a = `datafmt "datafmt";` + + `int = "%d";` + + `ptr = *;` + + `array = *;` + + `nil = ;` + + `empty = *:nil;` + + `datafmt.T4 = "<" (x:empty x | "-") ">" ` + +const F4b = `datafmt "datafmt";` + + `int = "%d";` + + `ptr = *;` + + `array = *;` + + `nil = ;` + + `empty = *:nil;` + + `datafmt.T4 = "<" (a:empty {a / ", "} | "-") ">" ` + +func TestStruct4(t *testing.T) { + x := 7 + check(t, F4a, "<->", T4{nil, nil}) + check(t, F4a, "<7>", T4{&x, nil}) + check(t, F4b, "<->", T4{nil, nil}) + check(t, F4b, "<2, 3, 7>", T4{nil, []int{2, 3, 7}}) +} + + +// ---------------------------------------------------------------------------- +// Formatting a struct (documentation example) + +type Point struct { + name string + x, y int +} + +const FPoint = `datafmt "datafmt";` + + `int = "%d";` + + `hexInt = "0x%x";` + + `string = "---%s---";` + + `datafmt.Point = name "{" x ", " y:hexInt "}";` + +func TestStructPoint(t *testing.T) { + p := Point{"foo", 3, 15} + check(t, FPoint, "---foo---{3, 0xf}", p) +} + + +// ---------------------------------------------------------------------------- +// Formatting a slice (documentation example) + +const FSlice = `int = "%b";` + + `array = { * / ", " }` + +func TestSlice(t *testing.T) { check(t, FSlice, "10, 11, 101, 111", []int{2, 3, 5, 7}) } + + +// TODO add more tests diff --git a/libgo/go/exp/datafmt/parser.go b/libgo/go/exp/datafmt/parser.go new file mode 100644 index 000000000..c6d140264 --- /dev/null +++ b/libgo/go/exp/datafmt/parser.go @@ -0,0 +1,386 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package datafmt + +import ( + "container/vector" + "go/scanner" + "go/token" + "os" + "strconv" + "strings" +) + +// ---------------------------------------------------------------------------- +// Parsing + +type parser struct { + scanner.ErrorVector + scanner scanner.Scanner + file *token.File + pos token.Pos // token position + tok token.Token // one token look-ahead + lit []byte // token literal + + packs map[string]string // PackageName -> ImportPath + rules map[string]expr // RuleName -> Expression +} + + +func (p *parser) next() { + p.pos, p.tok, p.lit = p.scanner.Scan() + switch p.tok { + case token.CHAN, token.FUNC, token.INTERFACE, token.MAP, token.STRUCT: + // Go keywords for composite types are type names + // returned by reflect. Accept them as identifiers. + p.tok = token.IDENT // p.lit is already set correctly + } +} + + +func (p *parser) init(fset *token.FileSet, filename string, src []byte) { + p.ErrorVector.Reset() + p.file = fset.AddFile(filename, fset.Base(), len(src)) + p.scanner.Init(p.file, src, p, scanner.AllowIllegalChars) // return '@' as token.ILLEGAL w/o error message + p.next() // initializes pos, tok, lit + p.packs = make(map[string]string) + p.rules = make(map[string]expr) +} + + +func (p *parser) error(pos token.Pos, msg string) { + p.Error(p.file.Position(pos), msg) +} + + +func (p *parser) errorExpected(pos token.Pos, msg string) { + msg = "expected " + msg + if pos == p.pos { + // the error happened at the current position; + // make the error message more specific + msg += ", found '" + p.tok.String() + "'" + if p.tok.IsLiteral() { + msg += " " + string(p.lit) + } + } + p.error(pos, msg) +} + + +func (p *parser) expect(tok token.Token) token.Pos { + pos := p.pos + if p.tok != tok { + p.errorExpected(pos, "'"+tok.String()+"'") + } + p.next() // make progress in any case + return pos +} + + +func (p *parser) parseIdentifier() string { + name := string(p.lit) + p.expect(token.IDENT) + return name +} + + +func (p *parser) parseTypeName() (string, bool) { + pos := p.pos + name, isIdent := p.parseIdentifier(), true + if p.tok == token.PERIOD { + // got a package name, lookup package + if importPath, found := p.packs[name]; found { + name = importPath + } else { + p.error(pos, "package not declared: "+name) + } + p.next() + name, isIdent = name+"."+p.parseIdentifier(), false + } + return name, isIdent +} + + +// Parses a rule name and returns it. If the rule name is +// a package-qualified type name, the package name is resolved. +// The 2nd result value is true iff the rule name consists of a +// single identifier only (and thus could be a package name). +// +func (p *parser) parseRuleName() (string, bool) { + name, isIdent := "", false + switch p.tok { + case token.IDENT: + name, isIdent = p.parseTypeName() + case token.DEFAULT: + name = "default" + p.next() + case token.QUO: + name = "/" + p.next() + default: + p.errorExpected(p.pos, "rule name") + p.next() // make progress in any case + } + return name, isIdent +} + + +func (p *parser) parseString() string { + s := "" + if p.tok == token.STRING { + s, _ = strconv.Unquote(string(p.lit)) + // Unquote may fail with an error, but only if the scanner found + // an illegal string in the first place. In this case the error + // has already been reported. + p.next() + return s + } else { + p.expect(token.STRING) + } + return s +} + + +func (p *parser) parseLiteral() literal { + s := []byte(p.parseString()) + + // A string literal may contain %-format specifiers. To simplify + // and speed up printing of the literal, split it into segments + // that start with "%" possibly followed by a last segment that + // starts with some other character. + var list vector.Vector + i0 := 0 + for i := 0; i < len(s); i++ { + if s[i] == '%' && i+1 < len(s) { + // the next segment starts with a % format + if i0 < i { + // the current segment is not empty, split it off + list.Push(s[i0:i]) + i0 = i + } + i++ // skip %; let loop skip over char after % + } + } + // the final segment may start with any character + // (it is empty iff the string is empty) + list.Push(s[i0:]) + + // convert list into a literal + lit := make(literal, list.Len()) + for i := 0; i < list.Len(); i++ { + lit[i] = list.At(i).([]byte) + } + + return lit +} + + +func (p *parser) parseField() expr { + var fname string + switch p.tok { + case token.ILLEGAL: + if string(p.lit) != "@" { + return nil + } + fname = "@" + p.next() + case token.MUL: + fname = "*" + p.next() + case token.IDENT: + fname = p.parseIdentifier() + default: + return nil + } + + var ruleName string + if p.tok == token.COLON { + p.next() + ruleName, _ = p.parseRuleName() + } + + return &field{fname, ruleName} +} + + +func (p *parser) parseOperand() (x expr) { + switch p.tok { + case token.STRING: + x = p.parseLiteral() + + case token.LPAREN: + p.next() + x = p.parseExpression() + if p.tok == token.SHR { + p.next() + x = &group{x, p.parseExpression()} + } + p.expect(token.RPAREN) + + case token.LBRACK: + p.next() + x = &option{p.parseExpression()} + p.expect(token.RBRACK) + + case token.LBRACE: + p.next() + x = p.parseExpression() + var div expr + if p.tok == token.QUO { + p.next() + div = p.parseExpression() + } + x = &repetition{x, div} + p.expect(token.RBRACE) + + default: + x = p.parseField() // may be nil + } + + return x +} + + +func (p *parser) parseSequence() expr { + var list vector.Vector + + for x := p.parseOperand(); x != nil; x = p.parseOperand() { + list.Push(x) + } + + // no need for a sequence if list.Len() < 2 + switch list.Len() { + case 0: + return nil + case 1: + return list.At(0).(expr) + } + + // convert list into a sequence + seq := make(sequence, list.Len()) + for i := 0; i < list.Len(); i++ { + seq[i] = list.At(i).(expr) + } + return seq +} + + +func (p *parser) parseExpression() expr { + var list vector.Vector + + for { + x := p.parseSequence() + if x != nil { + list.Push(x) + } + if p.tok != token.OR { + break + } + p.next() + } + + // no need for an alternatives if list.Len() < 2 + switch list.Len() { + case 0: + return nil + case 1: + return list.At(0).(expr) + } + + // convert list into a alternatives + alt := make(alternatives, list.Len()) + for i := 0; i < list.Len(); i++ { + alt[i] = list.At(i).(expr) + } + return alt +} + + +func (p *parser) parseFormat() { + for p.tok != token.EOF { + pos := p.pos + + name, isIdent := p.parseRuleName() + switch p.tok { + case token.STRING: + // package declaration + importPath := p.parseString() + + // add package declaration + if !isIdent { + p.error(pos, "illegal package name: "+name) + } else if _, found := p.packs[name]; !found { + p.packs[name] = importPath + } else { + p.error(pos, "package already declared: "+name) + } + + case token.ASSIGN: + // format rule + p.next() + x := p.parseExpression() + + // add rule + if _, found := p.rules[name]; !found { + p.rules[name] = x + } else { + p.error(pos, "format rule already declared: "+name) + } + + default: + p.errorExpected(p.pos, "package declaration or format rule") + p.next() // make progress in any case + } + + if p.tok == token.SEMICOLON { + p.next() + } else { + break + } + } + p.expect(token.EOF) +} + + +func remap(p *parser, name string) string { + i := strings.Index(name, ".") + if i >= 0 { + packageName, suffix := name[0:i], name[i:] + // lookup package + if importPath, found := p.packs[packageName]; found { + name = importPath + suffix + } else { + var invalidPos token.Position + p.Error(invalidPos, "package not declared: "+packageName) + } + } + return name +} + + +// Parse parses a set of format productions from source src. Custom +// formatters may be provided via a map of formatter functions. If +// there are no errors, the result is a Format and the error is nil. +// Otherwise the format is nil and a non-empty ErrorList is returned. +// +func Parse(fset *token.FileSet, filename string, src []byte, fmap FormatterMap) (Format, os.Error) { + // parse source + var p parser + p.init(fset, filename, src) + p.parseFormat() + + // add custom formatters, if any + for name, form := range fmap { + name = remap(&p, name) + if _, found := p.rules[name]; !found { + p.rules[name] = &custom{name, form} + } else { + var invalidPos token.Position + p.Error(invalidPos, "formatter already declared: "+name) + } + } + + return p.rules, p.GetError(scanner.NoMultiples) +} diff --git a/libgo/go/exp/draw/draw.go b/libgo/go/exp/draw/draw.go new file mode 100644 index 000000000..1d0729d92 --- /dev/null +++ b/libgo/go/exp/draw/draw.go @@ -0,0 +1,363 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package draw provides basic graphics and drawing primitives, +// in the style of the Plan 9 graphics library +// (see http://plan9.bell-labs.com/magic/man2html/2/draw) +// and the X Render extension. +package draw + +import "image" + +// m is the maximum color value returned by image.Color.RGBA. +const m = 1<<16 - 1 + +// A Porter-Duff compositing operator. +type Op int + +const ( + // Over specifies ``(src in mask) over dst''. + Over Op = iota + // Src specifies ``src in mask''. + Src +) + +var zeroColor image.Color = image.AlphaColor{0} + +// A draw.Image is an image.Image with a Set method to change a single pixel. +type Image interface { + image.Image + Set(x, y int, c image.Color) +} + +// Draw calls DrawMask with a nil mask and an Over op. +func Draw(dst Image, r image.Rectangle, src image.Image, sp image.Point) { + DrawMask(dst, r, src, sp, nil, image.ZP, Over) +} + +// DrawMask aligns r.Min in dst with sp in src and mp in mask and then replaces the rectangle r +// in dst with the result of a Porter-Duff composition. A nil mask is treated as opaque. +func DrawMask(dst Image, r image.Rectangle, src image.Image, sp image.Point, mask image.Image, mp image.Point, op Op) { + sb := src.Bounds() + dx, dy := sb.Max.X-sp.X, sb.Max.Y-sp.Y + if mask != nil { + mb := mask.Bounds() + if dx > mb.Max.X-mp.X { + dx = mb.Max.X - mp.X + } + if dy > mb.Max.Y-mp.Y { + dy = mb.Max.Y - mp.Y + } + } + if r.Dx() > dx { + r.Max.X = r.Min.X + dx + } + if r.Dy() > dy { + r.Max.Y = r.Min.Y + dy + } + r = r.Intersect(dst.Bounds()) + if r.Empty() { + return + } + + // Fast paths for special cases. If none of them apply, then we fall back to a general but slow implementation. + if dst0, ok := dst.(*image.RGBA); ok { + if op == Over { + if mask == nil { + if src0, ok := src.(*image.ColorImage); ok { + drawFillOver(dst0, r, src0) + return + } + if src0, ok := src.(*image.RGBA); ok { + drawCopyOver(dst0, r, src0, sp) + return + } + } else if mask0, ok := mask.(*image.Alpha); ok { + if src0, ok := src.(*image.ColorImage); ok { + drawGlyphOver(dst0, r, src0, mask0, mp) + return + } + } + } else { + if mask == nil { + if src0, ok := src.(*image.ColorImage); ok { + drawFillSrc(dst0, r, src0) + return + } + if src0, ok := src.(*image.RGBA); ok { + drawCopySrc(dst0, r, src0, sp) + return + } + } + } + drawRGBA(dst0, r, src, sp, mask, mp, op) + return + } + + x0, x1, dx := r.Min.X, r.Max.X, 1 + y0, y1, dy := r.Min.Y, r.Max.Y, 1 + if image.Image(dst) == src && r.Overlaps(r.Add(sp.Sub(r.Min))) { + // Rectangles overlap: process backward? + if sp.Y < r.Min.Y || sp.Y == r.Min.Y && sp.X < r.Min.X { + x0, x1, dx = x1-1, x0-1, -1 + y0, y1, dy = y1-1, y0-1, -1 + } + } + + var out *image.RGBA64Color + sy := sp.Y + y0 - r.Min.Y + my := mp.Y + y0 - r.Min.Y + for y := y0; y != y1; y, sy, my = y+dy, sy+dy, my+dy { + sx := sp.X + x0 - r.Min.X + mx := mp.X + x0 - r.Min.X + for x := x0; x != x1; x, sx, mx = x+dx, sx+dx, mx+dx { + ma := uint32(m) + if mask != nil { + _, _, _, ma = mask.At(mx, my).RGBA() + } + switch { + case ma == 0: + if op == Over { + // No-op. + } else { + dst.Set(x, y, zeroColor) + } + case ma == m && op == Src: + dst.Set(x, y, src.At(sx, sy)) + default: + sr, sg, sb, sa := src.At(sx, sy).RGBA() + if out == nil { + out = new(image.RGBA64Color) + } + if op == Over { + dr, dg, db, da := dst.At(x, y).RGBA() + a := m - (sa * ma / m) + out.R = uint16((dr*a + sr*ma) / m) + out.G = uint16((dg*a + sg*ma) / m) + out.B = uint16((db*a + sb*ma) / m) + out.A = uint16((da*a + sa*ma) / m) + } else { + out.R = uint16(sr * ma / m) + out.G = uint16(sg * ma / m) + out.B = uint16(sb * ma / m) + out.A = uint16(sa * ma / m) + } + dst.Set(x, y, out) + } + } + } +} + +func drawFillOver(dst *image.RGBA, r image.Rectangle, src *image.ColorImage) { + cr, cg, cb, ca := src.RGBA() + // The 0x101 is here for the same reason as in drawRGBA. + a := (m - ca) * 0x101 + x0, x1 := r.Min.X, r.Max.X + y0, y1 := r.Min.Y, r.Max.Y + for y := y0; y != y1; y++ { + dbase := y * dst.Stride + dpix := dst.Pix[dbase+x0 : dbase+x1] + for i, rgba := range dpix { + dr := (uint32(rgba.R)*a)/m + cr + dg := (uint32(rgba.G)*a)/m + cg + db := (uint32(rgba.B)*a)/m + cb + da := (uint32(rgba.A)*a)/m + ca + dpix[i] = image.RGBAColor{uint8(dr >> 8), uint8(dg >> 8), uint8(db >> 8), uint8(da >> 8)} + } + } +} + +func drawCopyOver(dst *image.RGBA, r image.Rectangle, src *image.RGBA, sp image.Point) { + dx0, dx1 := r.Min.X, r.Max.X + dy0, dy1 := r.Min.Y, r.Max.Y + nrows := dy1 - dy0 + sx0, sx1 := sp.X, sp.X+dx1-dx0 + d0 := dy0*dst.Stride + dx0 + d1 := dy0*dst.Stride + dx1 + s0 := sp.Y*src.Stride + sx0 + s1 := sp.Y*src.Stride + sx1 + var ( + ddelta, sdelta int + i0, i1, idelta int + ) + if r.Min.Y < sp.Y || r.Min.Y == sp.Y && r.Min.X <= sp.X { + ddelta = dst.Stride + sdelta = src.Stride + i0, i1, idelta = 0, d1-d0, +1 + } else { + // If the source start point is higher than the destination start point, or equal height but to the left, + // then we compose the rows in right-to-left, bottom-up order instead of left-to-right, top-down. + d0 += (nrows - 1) * dst.Stride + d1 += (nrows - 1) * dst.Stride + s0 += (nrows - 1) * src.Stride + s1 += (nrows - 1) * src.Stride + ddelta = -dst.Stride + sdelta = -src.Stride + i0, i1, idelta = d1-d0-1, -1, -1 + } + for ; nrows > 0; nrows-- { + dpix := dst.Pix[d0:d1] + spix := src.Pix[s0:s1] + for i := i0; i != i1; i += idelta { + // For unknown reasons, even though both dpix[i] and spix[i] are + // image.RGBAColors, on an x86 CPU it seems fastest to call RGBA + // for the source but to do it manually for the destination. + sr, sg, sb, sa := spix[i].RGBA() + rgba := dpix[i] + dr := uint32(rgba.R) + dg := uint32(rgba.G) + db := uint32(rgba.B) + da := uint32(rgba.A) + // The 0x101 is here for the same reason as in drawRGBA. + a := (m - sa) * 0x101 + dr = (dr*a)/m + sr + dg = (dg*a)/m + sg + db = (db*a)/m + sb + da = (da*a)/m + sa + dpix[i] = image.RGBAColor{uint8(dr >> 8), uint8(dg >> 8), uint8(db >> 8), uint8(da >> 8)} + } + d0 += ddelta + d1 += ddelta + s0 += sdelta + s1 += sdelta + } +} + +func drawGlyphOver(dst *image.RGBA, r image.Rectangle, src *image.ColorImage, mask *image.Alpha, mp image.Point) { + x0, x1 := r.Min.X, r.Max.X + y0, y1 := r.Min.Y, r.Max.Y + cr, cg, cb, ca := src.RGBA() + for y, my := y0, mp.Y; y != y1; y, my = y+1, my+1 { + dbase := y * dst.Stride + dpix := dst.Pix[dbase+x0 : dbase+x1] + mbase := my * mask.Stride + mpix := mask.Pix[mbase+mp.X:] + for i, rgba := range dpix { + ma := uint32(mpix[i].A) + if ma == 0 { + continue + } + ma |= ma << 8 + dr := uint32(rgba.R) + dg := uint32(rgba.G) + db := uint32(rgba.B) + da := uint32(rgba.A) + // The 0x101 is here for the same reason as in drawRGBA. + a := (m - (ca * ma / m)) * 0x101 + dr = (dr*a + cr*ma) / m + dg = (dg*a + cg*ma) / m + db = (db*a + cb*ma) / m + da = (da*a + ca*ma) / m + dpix[i] = image.RGBAColor{uint8(dr >> 8), uint8(dg >> 8), uint8(db >> 8), uint8(da >> 8)} + } + } +} + +func drawFillSrc(dst *image.RGBA, r image.Rectangle, src *image.ColorImage) { + if r.Dy() < 1 { + return + } + cr, cg, cb, ca := src.RGBA() + color := image.RGBAColor{uint8(cr >> 8), uint8(cg >> 8), uint8(cb >> 8), uint8(ca >> 8)} + // The built-in copy function is faster than a straightforward for loop to fill the destination with + // the color, but copy requires a slice source. We therefore use a for loop to fill the first row, and + // then use the first row as the slice source for the remaining rows. + dx0, dx1 := r.Min.X, r.Max.X + dy0, dy1 := r.Min.Y, r.Max.Y + dbase := dy0 * dst.Stride + i0, i1 := dbase+dx0, dbase+dx1 + firstRow := dst.Pix[i0:i1] + for i := range firstRow { + firstRow[i] = color + } + for y := dy0 + 1; y < dy1; y++ { + i0 += dst.Stride + i1 += dst.Stride + copy(dst.Pix[i0:i1], firstRow) + } +} + +func drawCopySrc(dst *image.RGBA, r image.Rectangle, src *image.RGBA, sp image.Point) { + dx0, dx1 := r.Min.X, r.Max.X + dy0, dy1 := r.Min.Y, r.Max.Y + nrows := dy1 - dy0 + sx0, sx1 := sp.X, sp.X+dx1-dx0 + d0 := dy0*dst.Stride + dx0 + d1 := dy0*dst.Stride + dx1 + s0 := sp.Y*src.Stride + sx0 + s1 := sp.Y*src.Stride + sx1 + var ddelta, sdelta int + if r.Min.Y <= sp.Y { + ddelta = dst.Stride + sdelta = src.Stride + } else { + // If the source start point is higher than the destination start point, then we compose the rows + // in bottom-up order instead of top-down. Unlike the drawCopyOver function, we don't have to + // check the x co-ordinates because the built-in copy function can handle overlapping slices. + d0 += (nrows - 1) * dst.Stride + d1 += (nrows - 1) * dst.Stride + s0 += (nrows - 1) * src.Stride + s1 += (nrows - 1) * src.Stride + ddelta = -dst.Stride + sdelta = -src.Stride + } + for ; nrows > 0; nrows-- { + copy(dst.Pix[d0:d1], src.Pix[s0:s1]) + d0 += ddelta + d1 += ddelta + s0 += sdelta + s1 += sdelta + } +} + +func drawRGBA(dst *image.RGBA, r image.Rectangle, src image.Image, sp image.Point, mask image.Image, mp image.Point, op Op) { + x0, x1, dx := r.Min.X, r.Max.X, 1 + y0, y1, dy := r.Min.Y, r.Max.Y, 1 + if image.Image(dst) == src && r.Overlaps(r.Add(sp.Sub(r.Min))) { + if sp.Y < r.Min.Y || sp.Y == r.Min.Y && sp.X < r.Min.X { + x0, x1, dx = x1-1, x0-1, -1 + y0, y1, dy = y1-1, y0-1, -1 + } + } + + sy := sp.Y + y0 - r.Min.Y + my := mp.Y + y0 - r.Min.Y + for y := y0; y != y1; y, sy, my = y+dy, sy+dy, my+dy { + sx := sp.X + x0 - r.Min.X + mx := mp.X + x0 - r.Min.X + dpix := dst.Pix[y*dst.Stride : (y+1)*dst.Stride] + for x := x0; x != x1; x, sx, mx = x+dx, sx+dx, mx+dx { + ma := uint32(m) + if mask != nil { + _, _, _, ma = mask.At(mx, my).RGBA() + } + sr, sg, sb, sa := src.At(sx, sy).RGBA() + var dr, dg, db, da uint32 + if op == Over { + rgba := dpix[x] + dr = uint32(rgba.R) + dg = uint32(rgba.G) + db = uint32(rgba.B) + da = uint32(rgba.A) + // dr, dg, db and da are all 8-bit color at the moment, ranging in [0,255]. + // We work in 16-bit color, and so would normally do: + // dr |= dr << 8 + // and similarly for dg, db and da, but instead we multiply a + // (which is a 16-bit color, ranging in [0,65535]) by 0x101. + // This yields the same result, but is fewer arithmetic operations. + a := (m - (sa * ma / m)) * 0x101 + dr = (dr*a + sr*ma) / m + dg = (dg*a + sg*ma) / m + db = (db*a + sb*ma) / m + da = (da*a + sa*ma) / m + } else { + dr = sr * ma / m + dg = sg * ma / m + db = sb * ma / m + da = sa * ma / m + } + dpix[x] = image.RGBAColor{uint8(dr >> 8), uint8(dg >> 8), uint8(db >> 8), uint8(da >> 8)} + } + } +} diff --git a/libgo/go/exp/draw/draw_test.go b/libgo/go/exp/draw/draw_test.go new file mode 100644 index 000000000..90c9e823d --- /dev/null +++ b/libgo/go/exp/draw/draw_test.go @@ -0,0 +1,228 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package draw + +import ( + "image" + "testing" +) + +func eq(c0, c1 image.Color) bool { + r0, g0, b0, a0 := c0.RGBA() + r1, g1, b1, a1 := c1.RGBA() + return r0 == r1 && g0 == g1 && b0 == b1 && a0 == a1 +} + +func fillBlue(alpha int) image.Image { + return image.NewColorImage(image.RGBAColor{0, 0, uint8(alpha), uint8(alpha)}) +} + +func fillAlpha(alpha int) image.Image { + return image.NewColorImage(image.AlphaColor{uint8(alpha)}) +} + +func vgradGreen(alpha int) image.Image { + m := image.NewRGBA(16, 16) + for y := 0; y < 16; y++ { + for x := 0; x < 16; x++ { + m.Set(x, y, image.RGBAColor{0, uint8(y * alpha / 15), 0, uint8(alpha)}) + } + } + return m +} + +func vgradAlpha(alpha int) image.Image { + m := image.NewAlpha(16, 16) + for y := 0; y < 16; y++ { + for x := 0; x < 16; x++ { + m.Set(x, y, image.AlphaColor{uint8(y * alpha / 15)}) + } + } + return m +} + +func hgradRed(alpha int) Image { + m := image.NewRGBA(16, 16) + for y := 0; y < 16; y++ { + for x := 0; x < 16; x++ { + m.Set(x, y, image.RGBAColor{uint8(x * alpha / 15), 0, 0, uint8(alpha)}) + } + } + return m +} + +func gradYellow(alpha int) Image { + m := image.NewRGBA(16, 16) + for y := 0; y < 16; y++ { + for x := 0; x < 16; x++ { + m.Set(x, y, image.RGBAColor{uint8(x * alpha / 15), uint8(y * alpha / 15), 0, uint8(alpha)}) + } + } + return m +} + +type drawTest struct { + desc string + src image.Image + mask image.Image + op Op + expected image.Color +} + +var drawTests = []drawTest{ + // Uniform mask (0% opaque). + {"nop", vgradGreen(255), fillAlpha(0), Over, image.RGBAColor{136, 0, 0, 255}}, + {"clear", vgradGreen(255), fillAlpha(0), Src, image.RGBAColor{0, 0, 0, 0}}, + // Uniform mask (100%, 75%, nil) and uniform source. + // At (x, y) == (8, 8): + // The destination pixel is {136, 0, 0, 255}. + // The source pixel is {0, 0, 90, 90}. + {"fill", fillBlue(90), fillAlpha(255), Over, image.RGBAColor{88, 0, 90, 255}}, + {"fillSrc", fillBlue(90), fillAlpha(255), Src, image.RGBAColor{0, 0, 90, 90}}, + {"fillAlpha", fillBlue(90), fillAlpha(192), Over, image.RGBAColor{100, 0, 68, 255}}, + {"fillAlphaSrc", fillBlue(90), fillAlpha(192), Src, image.RGBAColor{0, 0, 68, 68}}, + {"fillNil", fillBlue(90), nil, Over, image.RGBAColor{88, 0, 90, 255}}, + {"fillNilSrc", fillBlue(90), nil, Src, image.RGBAColor{0, 0, 90, 90}}, + // Uniform mask (100%, 75%, nil) and variable source. + // At (x, y) == (8, 8): + // The destination pixel is {136, 0, 0, 255}. + // The source pixel is {0, 48, 0, 90}. + {"copy", vgradGreen(90), fillAlpha(255), Over, image.RGBAColor{88, 48, 0, 255}}, + {"copySrc", vgradGreen(90), fillAlpha(255), Src, image.RGBAColor{0, 48, 0, 90}}, + {"copyAlpha", vgradGreen(90), fillAlpha(192), Over, image.RGBAColor{100, 36, 0, 255}}, + {"copyAlphaSrc", vgradGreen(90), fillAlpha(192), Src, image.RGBAColor{0, 36, 0, 68}}, + {"copyNil", vgradGreen(90), nil, Over, image.RGBAColor{88, 48, 0, 255}}, + {"copyNilSrc", vgradGreen(90), nil, Src, image.RGBAColor{0, 48, 0, 90}}, + // Variable mask and variable source. + // At (x, y) == (8, 8): + // The destination pixel is {136, 0, 0, 255}. + // The source pixel is {0, 0, 255, 255}. + // The mask pixel's alpha is 102, or 40%. + {"generic", fillBlue(255), vgradAlpha(192), Over, image.RGBAColor{81, 0, 102, 255}}, + {"genericSrc", fillBlue(255), vgradAlpha(192), Src, image.RGBAColor{0, 0, 102, 102}}, +} + +func makeGolden(dst, src, mask image.Image, op Op) image.Image { + // Since golden is a newly allocated image, we don't have to check if the + // input source and mask images and the output golden image overlap. + b := dst.Bounds() + sx0 := src.Bounds().Min.X - b.Min.X + sy0 := src.Bounds().Min.Y - b.Min.Y + var mx0, my0 int + if mask != nil { + mx0 = mask.Bounds().Min.X - b.Min.X + my0 = mask.Bounds().Min.Y - b.Min.Y + } + golden := image.NewRGBA(b.Max.X, b.Max.Y) + for y := b.Min.Y; y < b.Max.Y; y++ { + my, sy := my0+y, sy0+y + for x := b.Min.X; x < b.Max.X; x++ { + mx, sx := mx0+x, sx0+x + const M = 1<<16 - 1 + var dr, dg, db, da uint32 + if op == Over { + dr, dg, db, da = dst.At(x, y).RGBA() + } + sr, sg, sb, sa := src.At(sx, sy).RGBA() + ma := uint32(M) + if mask != nil { + _, _, _, ma = mask.At(mx, my).RGBA() + } + a := M - (sa * ma / M) + golden.Set(x, y, image.RGBA64Color{ + uint16((dr*a + sr*ma) / M), + uint16((dg*a + sg*ma) / M), + uint16((db*a + sb*ma) / M), + uint16((da*a + sa*ma) / M), + }) + } + } + golden.Rect = b + return golden +} + +func TestDraw(t *testing.T) { +loop: + for _, test := range drawTests { + dst := hgradRed(255) + // Draw the (src, mask, op) onto a copy of dst using a slow but obviously correct implementation. + golden := makeGolden(dst, test.src, test.mask, test.op) + b := dst.Bounds() + if !b.Eq(golden.Bounds()) { + t.Errorf("draw %s: bounds %v versus %v", test.desc, dst.Bounds(), golden.Bounds()) + continue + } + // Draw the same combination onto the actual dst using the optimized DrawMask implementation. + DrawMask(dst, b, test.src, image.ZP, test.mask, image.ZP, test.op) + // Check that the resultant pixel at (8, 8) matches what we expect + // (the expected value can be verified by hand). + if !eq(dst.At(8, 8), test.expected) { + t.Errorf("draw %s: at (8, 8) %v versus %v", test.desc, dst.At(8, 8), test.expected) + continue + } + // Check that the resultant dst image matches the golden output. + for y := b.Min.Y; y < b.Max.Y; y++ { + for x := b.Min.X; x < b.Max.X; x++ { + if !eq(dst.At(x, y), golden.At(x, y)) { + t.Errorf("draw %s: at (%d, %d), %v versus golden %v", test.desc, x, y, dst.At(x, y), golden.At(x, y)) + continue loop + } + } + } + } +} + +func TestDrawOverlap(t *testing.T) { + for _, op := range []Op{Over, Src} { + for yoff := -2; yoff <= 2; yoff++ { + loop: + for xoff := -2; xoff <= 2; xoff++ { + m := gradYellow(127).(*image.RGBA) + dst := &image.RGBA{ + Pix: m.Pix, + Stride: m.Stride, + Rect: image.Rect(5, 5, 10, 10), + } + src := &image.RGBA{ + Pix: m.Pix, + Stride: m.Stride, + Rect: image.Rect(5+xoff, 5+yoff, 10+xoff, 10+yoff), + } + // Draw the (src, mask, op) onto a copy of dst using a slow but obviously correct implementation. + golden := makeGolden(dst, src, nil, op) + b := dst.Bounds() + if !b.Eq(golden.Bounds()) { + t.Errorf("drawOverlap xoff=%d,yoff=%d: bounds %v versus %v", xoff, yoff, dst.Bounds(), golden.Bounds()) + continue + } + // Draw the same combination onto the actual dst using the optimized DrawMask implementation. + DrawMask(dst, b, src, src.Bounds().Min, nil, image.ZP, op) + // Check that the resultant dst image matches the golden output. + for y := b.Min.Y; y < b.Max.Y; y++ { + for x := b.Min.X; x < b.Max.X; x++ { + if !eq(dst.At(x, y), golden.At(x, y)) { + t.Errorf("drawOverlap xoff=%d,yoff=%d: at (%d, %d), %v versus golden %v", xoff, yoff, x, y, dst.At(x, y), golden.At(x, y)) + continue loop + } + } + } + } + } + } +} + +// TestIssue836 verifies http://code.google.com/p/go/issues/detail?id=836. +func TestIssue836(t *testing.T) { + a := image.NewRGBA(1, 1) + b := image.NewRGBA(2, 2) + b.Set(0, 0, image.RGBAColor{0, 0, 0, 5}) + b.Set(1, 0, image.RGBAColor{0, 0, 5, 5}) + b.Set(0, 1, image.RGBAColor{0, 5, 0, 5}) + b.Set(1, 1, image.RGBAColor{5, 0, 0, 5}) + Draw(a, image.Rect(0, 0, 1, 1), b, image.Pt(1, 1)) + if !eq(image.RGBAColor{5, 0, 0, 5}, a.At(0, 0)) { + t.Errorf("Issue 836: want %v got %v", image.RGBAColor{5, 0, 0, 5}, a.At(0, 0)) + } +} diff --git a/libgo/go/exp/draw/event.go b/libgo/go/exp/draw/event.go new file mode 100644 index 000000000..b777d912e --- /dev/null +++ b/libgo/go/exp/draw/event.go @@ -0,0 +1,56 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package draw + +import ( + "image" + "os" +) + +// A Window represents a single graphics window. +type Window interface { + // Screen returns an editable Image for the window. + Screen() Image + // FlushImage flushes changes made to Screen() back to screen. + FlushImage() + // EventChan returns a channel carrying UI events such as key presses, + // mouse movements and window resizes. + EventChan() <-chan interface{} + // Close closes the window. + Close() os.Error +} + +// A KeyEvent is sent for a key press or release. +type KeyEvent struct { + // The value k represents key k being pressed. + // The value -k represents key k being released. + // The specific set of key values is not specified, + // but ordinary characters represent themselves. + Key int +} + +// A MouseEvent is sent for a button press or release or for a mouse movement. +type MouseEvent struct { + // Buttons is a bit mask of buttons: 1<<0 is left, 1<<1 middle, 1<<2 right. + // It represents button state and not necessarily the state delta: bit 0 + // being on means that the left mouse button is down, but does not imply + // that the same button was up in the previous MouseEvent. + Buttons int + // Loc is the location of the cursor. + Loc image.Point + // Nsec is the event's timestamp. + Nsec int64 +} + +// A ConfigEvent is sent each time the window's color model or size changes. +// The client should respond by calling Window.Screen to obtain a new image. +type ConfigEvent struct { + Config image.Config +} + +// An ErrEvent is sent when an error occurs. +type ErrEvent struct { + Err os.Error +} diff --git a/libgo/go/exp/draw/x11/auth.go b/libgo/go/exp/draw/x11/auth.go new file mode 100644 index 000000000..896dedf05 --- /dev/null +++ b/libgo/go/exp/draw/x11/auth.go @@ -0,0 +1,93 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package x11 + +import ( + "bufio" + "io" + "os" +) + +// readU16BE reads a big-endian uint16 from r, using b as a scratch buffer. +func readU16BE(r io.Reader, b []byte) (uint16, os.Error) { + _, err := io.ReadFull(r, b[0:2]) + if err != nil { + return 0, err + } + return uint16(b[0])<<8 + uint16(b[1]), nil +} + +// readStr reads a length-prefixed string from r, using b as a scratch buffer. +func readStr(r io.Reader, b []byte) (string, os.Error) { + n, err := readU16BE(r, b) + if err != nil { + return "", err + } + if int(n) > len(b) { + return "", os.NewError("Xauthority entry too long for buffer") + } + _, err = io.ReadFull(r, b[0:n]) + if err != nil { + return "", err + } + return string(b[0:n]), nil +} + +// readAuth reads the X authority file and returns the name/data pair for the display. +// displayStr is the "12" out of a $DISPLAY like ":12.0". +func readAuth(displayStr string) (name, data string, err os.Error) { + // b is a scratch buffer to use and should be at least 256 bytes long + // (i.e. it should be able to hold a hostname). + var b [256]byte + // As per /usr/include/X11/Xauth.h. + const familyLocal = 256 + + fn := os.Getenv("XAUTHORITY") + if fn == "" { + home := os.Getenv("HOME") + if home == "" { + err = os.NewError("Xauthority not found: $XAUTHORITY, $HOME not set") + return + } + fn = home + "/.Xauthority" + } + r, err := os.Open(fn, os.O_RDONLY, 0444) + if err != nil { + return + } + defer r.Close() + br := bufio.NewReader(r) + + hostname, err := os.Hostname() + if err != nil { + return + } + for { + family, err := readU16BE(br, b[0:2]) + if err != nil { + return + } + addr, err := readStr(br, b[0:]) + if err != nil { + return + } + disp, err := readStr(br, b[0:]) + if err != nil { + return + } + name0, err := readStr(br, b[0:]) + if err != nil { + return + } + data0, err := readStr(br, b[0:]) + if err != nil { + return + } + if family == familyLocal && addr == hostname && disp == displayStr { + return name0, data0, nil + } + } + panic("unreachable") +} diff --git a/libgo/go/exp/draw/x11/conn.go b/libgo/go/exp/draw/x11/conn.go new file mode 100644 index 000000000..da2181536 --- /dev/null +++ b/libgo/go/exp/draw/x11/conn.go @@ -0,0 +1,622 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This package implements an X11 backend for the exp/draw package. +// +// The X protocol specification is at ftp://ftp.x.org/pub/X11R7.0/doc/PDF/proto.pdf. +// A summary of the wire format can be found in XCB's xproto.xml. +package x11 + +import ( + "bufio" + "exp/draw" + "image" + "io" + "log" + "net" + "os" + "strconv" + "strings" + "time" +) + +type resID uint32 // X resource IDs. + +// TODO(nigeltao): Handle window resizes. +const ( + windowHeight = 600 + windowWidth = 800 +) + +const ( + keymapLo = 8 + keymapHi = 255 +) + +type conn struct { + c io.Closer + r *bufio.Reader + w *bufio.Writer + + gc, window, root, visual resID + + img *image.RGBA + eventc chan interface{} + mouseState draw.MouseEvent + + buf [256]byte // General purpose scratch buffer. + + flush chan bool + flushBuf0 [24]byte + flushBuf1 [4 * 1024]byte +} + +// writeSocket runs in its own goroutine, serving both FlushImage calls +// directly from the exp/draw client and indirectly from X expose events. +// It paints c.img to the X server via PutImage requests. +func (c *conn) writeSocket() { + defer c.c.Close() + for _ = range c.flush { + b := c.img.Bounds() + if b.Empty() { + continue + } + // Each X request has a 16-bit length (in terms of 4-byte units). To avoid going over + // this limit, we send PutImage for each row of the image, rather than trying to paint + // the entire image in one X request. This approach could easily be optimized (or the + // X protocol may have an escape sequence to delimit very large requests). + // TODO(nigeltao): See what XCB's xcb_put_image does in this situation. + units := 6 + b.Dx() + if units > 0xffff || b.Dy() > 0xffff { + log.Print("x11: window is too large for PutImage") + return + } + + c.flushBuf0[0] = 0x48 // PutImage opcode. + c.flushBuf0[1] = 0x02 // XCB_IMAGE_FORMAT_Z_PIXMAP. + c.flushBuf0[2] = uint8(units) + c.flushBuf0[3] = uint8(units >> 8) + setU32LE(c.flushBuf0[4:8], uint32(c.window)) + setU32LE(c.flushBuf0[8:12], uint32(c.gc)) + setU32LE(c.flushBuf0[12:16], 1<<16|uint32(b.Dx())) + c.flushBuf0[21] = 0x18 // depth = 24 bits. + + for y := b.Min.Y; y < b.Max.Y; y++ { + setU32LE(c.flushBuf0[16:20], uint32(y<<16)) + if _, err := c.w.Write(c.flushBuf0[0:24]); err != nil { + if err != os.EOF { + log.Println("x11:", err.String()) + } + return + } + p := c.img.Pix[y*c.img.Stride : (y+1)*c.img.Stride] + for x := b.Min.X; x < b.Max.X; { + nx := b.Max.X - x + if nx > len(c.flushBuf1)/4 { + nx = len(c.flushBuf1) / 4 + } + for i, rgba := range p[x : x+nx] { + c.flushBuf1[4*i+0] = rgba.B + c.flushBuf1[4*i+1] = rgba.G + c.flushBuf1[4*i+2] = rgba.R + } + x += nx + if _, err := c.w.Write(c.flushBuf1[0 : 4*nx]); err != nil { + if err != os.EOF { + log.Println("x11:", err.String()) + } + return + } + } + } + if err := c.w.Flush(); err != nil { + if err != os.EOF { + log.Println("x11:", err.String()) + } + return + } + } +} + +func (c *conn) Screen() draw.Image { return c.img } + +func (c *conn) FlushImage() { + // We do the send (the <- operator) in an expression context, rather than in + // a statement context, so that it does not block, and fails if the buffered + // channel is full (in which case there already is a flush request pending). + _ = c.flush <- false +} + +func (c *conn) Close() os.Error { + // Shut down the writeSocket goroutine. This will close the socket to the + // X11 server, which will cause c.eventc to close. + close(c.flush) + for _ = range c.eventc { + // Drain the channel to allow the readSocket goroutine to shut down. + } + return nil +} + +func (c *conn) EventChan() <-chan interface{} { return c.eventc } + +// readSocket runs in its own goroutine, reading X events and sending draw +// events on c's EventChan. +func (c *conn) readSocket() { + var ( + keymap [256][]int + keysymsPerKeycode int + ) + defer close(c.eventc) + for { + // X events are always 32 bytes long. + if _, err := io.ReadFull(c.r, c.buf[0:32]); err != nil { + if err != os.EOF { + c.eventc <- draw.ErrEvent{err} + } + return + } + switch c.buf[0] { + case 0x01: // Reply from a request (e.g. GetKeyboardMapping). + cookie := int(c.buf[3])<<8 | int(c.buf[2]) + if cookie != 1 { + // We issued only one request (GetKeyboardMapping) with a cookie of 1, + // so we shouldn't get any other reply from the X server. + c.eventc <- draw.ErrEvent{os.NewError("x11: unexpected cookie")} + return + } + keysymsPerKeycode = int(c.buf[1]) + b := make([]int, 256*keysymsPerKeycode) + for i := range keymap { + keymap[i] = b[i*keysymsPerKeycode : (i+1)*keysymsPerKeycode] + } + for i := keymapLo; i <= keymapHi; i++ { + m := keymap[i] + for j := range m { + u, err := readU32LE(c.r, c.buf[0:4]) + if err != nil { + if err != os.EOF { + c.eventc <- draw.ErrEvent{err} + } + return + } + m[j] = int(u) + } + } + case 0x02, 0x03: // Key press, key release. + // X Keyboard Encoding is documented at http://tronche.com/gui/x/xlib/input/keyboard-encoding.html + // TODO(nigeltao): Do we need to implement the "MODE SWITCH / group modifier" feature + // or is that some no-longer-used X construct? + if keysymsPerKeycode < 2 { + // Either we haven't yet received the GetKeyboardMapping reply or + // the X server has sent one that's too short. + continue + } + keycode := int(c.buf[1]) + shift := int(c.buf[28]) & 0x01 + keysym := keymap[keycode][shift] + if keysym == 0 { + keysym = keymap[keycode][0] + } + // TODO(nigeltao): Should we send KeyEvents for Shift/Ctrl/Alt? Should Shift-A send + // the same int down the channel as the sent on just the A key? + // TODO(nigeltao): How should IME events (e.g. key presses that should generate CJK text) work? Or + // is that outside the scope of the draw.Window interface? + if c.buf[0] == 0x03 { + keysym = -keysym + } + c.eventc <- draw.KeyEvent{keysym} + case 0x04, 0x05: // Button press, button release. + mask := 1 << (c.buf[1] - 1) + if c.buf[0] == 0x04 { + c.mouseState.Buttons |= mask + } else { + c.mouseState.Buttons &^= mask + } + c.mouseState.Nsec = time.Nanoseconds() + c.eventc <- c.mouseState + case 0x06: // Motion notify. + c.mouseState.Loc.X = int(int16(c.buf[25])<<8 | int16(c.buf[24])) + c.mouseState.Loc.Y = int(int16(c.buf[27])<<8 | int16(c.buf[26])) + c.mouseState.Nsec = time.Nanoseconds() + c.eventc <- c.mouseState + case 0x0c: // Expose. + // A single user action could trigger multiple expose events (e.g. if moving another + // window with XShape'd rounded corners over our window). In that case, the X server will + // send a uint16 count (in bytes 16-17) of the number of additional expose events coming. + // We could parse each event for the (x, y, width, height) and maintain a minimal dirty + // rectangle, but for now, the simplest approach is to paint the entire window, when + // receiving the final event in the series. + if c.buf[17] == 0 && c.buf[16] == 0 { + // TODO(nigeltao): Should we ignore the very first expose event? A freshly mapped window + // will trigger expose, but until the first c.FlushImage call, there's probably nothing to + // paint but black. For an 800x600 window, at 4 bytes per pixel, each repaint writes about + // 2MB over the socket. + c.FlushImage() + } + // TODO(nigeltao): Should we listen to DestroyNotify (0x11) and ResizeRequest (0x19) events? + // What about EnterNotify (0x07) and LeaveNotify (0x08)? + } + } +} + +// connect connects to the X server given by the full X11 display name (e.g. +// ":12.0") and returns the connection as well as the portion of the full name +// that is the display number (e.g. "12"). +// Examples: +// connect(":1") // calls net.Dial("unix", "", "/tmp/.X11-unix/X1"), displayStr="1" +// connect("/tmp/launch-123/:0") // calls net.Dial("unix", "", "/tmp/launch-123/:0"), displayStr="0" +// connect("hostname:2.1") // calls net.Dial("tcp", "", "hostname:6002"), displayStr="2" +// connect("tcp/hostname:1.0") // calls net.Dial("tcp", "", "hostname:6001"), displayStr="1" +func connect(display string) (conn net.Conn, displayStr string, err os.Error) { + colonIdx := strings.LastIndex(display, ":") + if colonIdx < 0 { + return nil, "", os.NewError("bad display: " + display) + } + // Parse the section before the colon. + var protocol, host, socket string + if display[0] == '/' { + socket = display[0:colonIdx] + } else { + if i := strings.LastIndex(display, "/"); i < 0 { + // The default protocol is TCP. + protocol = "tcp" + host = display[0:colonIdx] + } else { + protocol = display[0:i] + host = display[i+1 : colonIdx] + } + } + // Parse the section after the colon. + after := display[colonIdx+1:] + if after == "" { + return nil, "", os.NewError("bad display: " + display) + } + if i := strings.LastIndex(after, "."); i < 0 { + displayStr = after + } else { + displayStr = after[0:i] + } + displayInt, err := strconv.Atoi(displayStr) + if err != nil || displayInt < 0 { + return nil, "", os.NewError("bad display: " + display) + } + // Make the connection. + if socket != "" { + conn, err = net.Dial("unix", "", socket+":"+displayStr) + } else if host != "" { + conn, err = net.Dial(protocol, "", host+":"+strconv.Itoa(6000+displayInt)) + } else { + conn, err = net.Dial("unix", "", "/tmp/.X11-unix/X"+displayStr) + } + if err != nil { + return nil, "", os.NewError("cannot connect to " + display + ": " + err.String()) + } + return +} + +// authenticate authenticates ourselves with the X server. +// displayStr is the "12" out of ":12.0". +func authenticate(w *bufio.Writer, displayStr string) os.Error { + key, value, err := readAuth(displayStr) + if err != nil { + return err + } + // Assume that the authentication protocol is "MIT-MAGIC-COOKIE-1". + if len(key) != 18 || len(value) != 16 { + return os.NewError("unsupported Xauth") + } + // 0x006c means little-endian. 0x000b, 0x0000 means X major version 11, minor version 0. + // 0x0012 and 0x0010 means the auth key and value have lenths 18 and 16. + // The final 0x0000 is padding, so that the string length is a multiple of 4. + _, err = io.WriteString(w, "\x6c\x00\x0b\x00\x00\x00\x12\x00\x10\x00\x00\x00") + if err != nil { + return err + } + _, err = io.WriteString(w, key) + if err != nil { + return err + } + // Again, the 0x0000 is padding. + _, err = io.WriteString(w, "\x00\x00") + if err != nil { + return err + } + _, err = io.WriteString(w, value) + if err != nil { + return err + } + err = w.Flush() + if err != nil { + return err + } + return nil +} + +// readU8 reads a uint8 from r, using b as a scratch buffer. +func readU8(r io.Reader, b []byte) (uint8, os.Error) { + _, err := io.ReadFull(r, b[0:1]) + if err != nil { + return 0, err + } + return uint8(b[0]), nil +} + +// readU16LE reads a little-endian uint16 from r, using b as a scratch buffer. +func readU16LE(r io.Reader, b []byte) (uint16, os.Error) { + _, err := io.ReadFull(r, b[0:2]) + if err != nil { + return 0, err + } + return uint16(b[0]) | uint16(b[1])<<8, nil +} + +// readU32LE reads a little-endian uint32 from r, using b as a scratch buffer. +func readU32LE(r io.Reader, b []byte) (uint32, os.Error) { + _, err := io.ReadFull(r, b[0:4]) + if err != nil { + return 0, err + } + return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24, nil +} + +// setU32LE sets b[0:4] to be the little-endian representation of u. +func setU32LE(b []byte, u uint32) { + b[0] = byte((u >> 0) & 0xff) + b[1] = byte((u >> 8) & 0xff) + b[2] = byte((u >> 16) & 0xff) + b[3] = byte((u >> 24) & 0xff) +} + +// checkPixmapFormats checks that we have an agreeable X pixmap Format. +func checkPixmapFormats(r io.Reader, b []byte, n int) (agree bool, err os.Error) { + for i := 0; i < n; i++ { + _, err = io.ReadFull(r, b[0:8]) + if err != nil { + return + } + // Byte 0 is depth, byte 1 is bits-per-pixel, byte 2 is scanline-pad, the rest (5) is padding. + if b[0] == 24 && b[1] == 32 { + agree = true + } + } + return +} + +// checkDepths checks that we have an agreeable X Depth (i.e. one that has an agreeable X VisualType). +func checkDepths(r io.Reader, b []byte, n int, visual uint32) (agree bool, err os.Error) { + for i := 0; i < n; i++ { + depth, err := readU16LE(r, b) + if err != nil { + return + } + depth &= 0xff + visualsLen, err := readU16LE(r, b) + if err != nil { + return + } + // Ignore 4 bytes of padding. + _, err = io.ReadFull(r, b[0:4]) + if err != nil { + return + } + for j := 0; j < int(visualsLen); j++ { + // Read 24 bytes: visual(4), class(1), bits per rgb value(1), colormap entries(2), + // red mask(4), green mask(4), blue mask(4), padding(4). + v, err := readU32LE(r, b) + _, err = readU32LE(r, b) + rm, err := readU32LE(r, b) + gm, err := readU32LE(r, b) + bm, err := readU32LE(r, b) + _, err = readU32LE(r, b) + if err != nil { + return + } + if v == visual && rm == 0xff0000 && gm == 0xff00 && bm == 0xff && depth == 24 { + agree = true + } + } + } + return +} + +// checkScreens checks that we have an agreeable X Screen. +func checkScreens(r io.Reader, b []byte, n int) (root, visual uint32, err os.Error) { + for i := 0; i < n; i++ { + root0, err := readU32LE(r, b) + if err != nil { + return + } + // Ignore the next 7x4 bytes, which is: colormap, whitepixel, blackpixel, current input masks, + // width and height (pixels), width and height (mm), min and max installed maps. + _, err = io.ReadFull(r, b[0:28]) + if err != nil { + return + } + visual0, err := readU32LE(r, b) + if err != nil { + return + } + // Next 4 bytes: backing stores, save unders, root depth, allowed depths length. + x, err := readU32LE(r, b) + if err != nil { + return + } + nDepths := int(x >> 24) + agree, err := checkDepths(r, b, nDepths, visual0) + if err != nil { + return + } + if agree && root == 0 { + root = root0 + visual = visual0 + } + } + return +} + +// handshake performs the protocol handshake with the X server, and ensures +// that the server provides a compatible Screen, Depth, etc. +func (c *conn) handshake() os.Error { + _, err := io.ReadFull(c.r, c.buf[0:8]) + if err != nil { + return err + } + // Byte 0:1 should be 1 (success), bytes 2:6 should be 0xb0000000 (major/minor version 11.0). + if c.buf[0] != 1 || c.buf[2] != 11 || c.buf[3] != 0 || c.buf[4] != 0 || c.buf[5] != 0 { + return os.NewError("unsupported X version") + } + // Ignore the release number. + _, err = io.ReadFull(c.r, c.buf[0:4]) + if err != nil { + return err + } + // Read the resource ID base. + resourceIdBase, err := readU32LE(c.r, c.buf[0:4]) + if err != nil { + return err + } + // Read the resource ID mask. + resourceIdMask, err := readU32LE(c.r, c.buf[0:4]) + if err != nil { + return err + } + if resourceIdMask < 256 { + return os.NewError("X resource ID mask is too small") + } + // Ignore the motion buffer size. + _, err = io.ReadFull(c.r, c.buf[0:4]) + if err != nil { + return err + } + // Read the vendor length and round it up to a multiple of 4, + // for X11 protocol alignment reasons. + vendorLen, err := readU16LE(c.r, c.buf[0:2]) + if err != nil { + return err + } + vendorLen = (vendorLen + 3) &^ 3 + // Read the maximum request length. + maxReqLen, err := readU16LE(c.r, c.buf[0:2]) + if err != nil { + return err + } + if maxReqLen != 0xffff { + return os.NewError("unsupported X maximum request length") + } + // Read the roots length. + rootsLen, err := readU8(c.r, c.buf[0:1]) + if err != nil { + return err + } + // Read the pixmap formats length. + pixmapFormatsLen, err := readU8(c.r, c.buf[0:1]) + if err != nil { + return err + } + // Ignore some things that we don't care about (totalling 10 + vendorLen bytes): + // imageByteOrder(1), bitmapFormatBitOrder(1), bitmapFormatScanlineUnit(1) bitmapFormatScanlinePad(1), + // minKeycode(1), maxKeycode(1), padding(4), vendor (vendorLen). + if 10+int(vendorLen) > cap(c.buf) { + return os.NewError("unsupported X vendor") + } + _, err = io.ReadFull(c.r, c.buf[0:10+int(vendorLen)]) + if err != nil { + return err + } + // Check that we have an agreeable pixmap format. + agree, err := checkPixmapFormats(c.r, c.buf[0:8], int(pixmapFormatsLen)) + if err != nil { + return err + } + if !agree { + return os.NewError("unsupported X pixmap formats") + } + // Check that we have an agreeable screen. + root, visual, err := checkScreens(c.r, c.buf[0:24], int(rootsLen)) + if err != nil { + return err + } + if root == 0 || visual == 0 { + return os.NewError("unsupported X screen") + } + c.gc = resID(resourceIdBase) + c.window = resID(resourceIdBase + 1) + c.root = resID(root) + c.visual = resID(visual) + return nil +} + +// NewWindow calls NewWindowDisplay with $DISPLAY. +func NewWindow() (draw.Window, os.Error) { + display := os.Getenv("DISPLAY") + if len(display) == 0 { + return nil, os.NewError("$DISPLAY not set") + } + return NewWindowDisplay(display) +} + +// NewWindowDisplay returns a new draw.Window, backed by a newly created and +// mapped X11 window. The X server to connect to is specified by the display +// string, such as ":1". +func NewWindowDisplay(display string) (draw.Window, os.Error) { + socket, displayStr, err := connect(display) + if err != nil { + return nil, err + } + c := new(conn) + c.c = socket + c.r = bufio.NewReader(socket) + c.w = bufio.NewWriter(socket) + err = authenticate(c.w, displayStr) + if err != nil { + return nil, err + } + err = c.handshake() + if err != nil { + return nil, err + } + + // Now that we're connected, show a window, via three X protocol messages. + // First, issue a GetKeyboardMapping request. This is the first request, and + // will be associated with a cookie of 1. + setU32LE(c.buf[0:4], 0x00020065) // 0x65 is the GetKeyboardMapping opcode, and the message is 2 x 4 bytes long. + setU32LE(c.buf[4:8], uint32((keymapHi-keymapLo+1)<<8|keymapLo)) + // Second, create a graphics context (GC). + setU32LE(c.buf[8:12], 0x00060037) // 0x37 is the CreateGC opcode, and the message is 6 x 4 bytes long. + setU32LE(c.buf[12:16], uint32(c.gc)) + setU32LE(c.buf[16:20], uint32(c.root)) + setU32LE(c.buf[20:24], 0x00010004) // Bit 2 is XCB_GC_FOREGROUND, bit 16 is XCB_GC_GRAPHICS_EXPOSURES. + setU32LE(c.buf[24:28], 0x00000000) // The Foreground is black. + setU32LE(c.buf[28:32], 0x00000000) // GraphicsExposures' value is unused. + // Third, create the window. + setU32LE(c.buf[32:36], 0x000a0001) // 0x01 is the CreateWindow opcode, and the message is 10 x 4 bytes long. + setU32LE(c.buf[36:40], uint32(c.window)) + setU32LE(c.buf[40:44], uint32(c.root)) + setU32LE(c.buf[44:48], 0x00000000) // Initial (x, y) is (0, 0). + setU32LE(c.buf[48:52], windowHeight<<16|windowWidth) + setU32LE(c.buf[52:56], 0x00010000) // Border width is 0, XCB_WINDOW_CLASS_INPUT_OUTPUT is 1. + setU32LE(c.buf[56:60], uint32(c.visual)) + setU32LE(c.buf[60:64], 0x00000802) // Bit 1 is XCB_CW_BACK_PIXEL, bit 11 is XCB_CW_EVENT_MASK. + setU32LE(c.buf[64:68], 0x00000000) // The Back-Pixel is black. + setU32LE(c.buf[68:72], 0x0000804f) // Key/button press and release, pointer motion, and expose event masks. + // Fourth, map the window. + setU32LE(c.buf[72:76], 0x00020008) // 0x08 is the MapWindow opcode, and the message is 2 x 4 bytes long. + setU32LE(c.buf[76:80], uint32(c.window)) + // Write the bytes. + _, err = c.w.Write(c.buf[0:80]) + if err != nil { + return nil, err + } + err = c.w.Flush() + if err != nil { + return nil, err + } + + c.img = image.NewRGBA(windowWidth, windowHeight) + c.eventc = make(chan interface{}, 16) + c.flush = make(chan bool, 1) + go c.readSocket() + go c.writeSocket() + return c, nil +} diff --git a/libgo/go/exp/eval/abort.go b/libgo/go/exp/eval/abort.go new file mode 100644 index 000000000..22e17cec4 --- /dev/null +++ b/libgo/go/exp/eval/abort.go @@ -0,0 +1,85 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package eval + +import ( + "fmt" + "os" + "runtime" +) + +// Abort aborts the thread's current computation, +// causing the innermost Try to return err. +func (t *Thread) Abort(err os.Error) { + if t.abort == nil { + panic("abort: " + err.String()) + } + t.abort <- err + runtime.Goexit() +} + +// Try executes a computation; if the computation +// Aborts, Try returns the error passed to abort. +func (t *Thread) Try(f func(t *Thread)) os.Error { + oc := t.abort + c := make(chan os.Error) + t.abort = c + go func() { + f(t) + c <- nil + }() + err := <-c + t.abort = oc + return err +} + +type DivByZeroError struct{} + +func (DivByZeroError) String() string { return "divide by zero" } + +type NilPointerError struct{} + +func (NilPointerError) String() string { return "nil pointer dereference" } + +type IndexError struct { + Idx, Len int64 +} + +func (e IndexError) String() string { + if e.Idx < 0 { + return fmt.Sprintf("negative index: %d", e.Idx) + } + return fmt.Sprintf("index %d exceeds length %d", e.Idx, e.Len) +} + +type SliceError struct { + Lo, Hi, Cap int64 +} + +func (e SliceError) String() string { + return fmt.Sprintf("slice [%d:%d]; cap %d", e.Lo, e.Hi, e.Cap) +} + +type KeyError struct { + Key interface{} +} + +func (e KeyError) String() string { return fmt.Sprintf("key '%v' not found in map", e.Key) } + +type NegativeLengthError struct { + Len int64 +} + +func (e NegativeLengthError) String() string { + return fmt.Sprintf("negative length: %d", e.Len) +} + +type NegativeCapacityError struct { + Len int64 +} + +func (e NegativeCapacityError) String() string { + return fmt.Sprintf("negative capacity: %d", e.Len) +} diff --git a/libgo/go/exp/eval/bridge.go b/libgo/go/exp/eval/bridge.go new file mode 100644 index 000000000..12835c4c0 --- /dev/null +++ b/libgo/go/exp/eval/bridge.go @@ -0,0 +1,169 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package eval + +import ( + "log" + "go/token" + "reflect" +) + +/* + * Type bridging + */ + +var ( + evalTypes = make(map[reflect.Type]Type) + nativeTypes = make(map[Type]reflect.Type) +) + +// TypeFromNative converts a regular Go type into a the corresponding +// interpreter Type. +func TypeFromNative(t reflect.Type) Type { + if et, ok := evalTypes[t]; ok { + return et + } + + var nt *NamedType + if t.Name() != "" { + name := t.PkgPath() + "·" + t.Name() + nt = &NamedType{token.NoPos, name, nil, true, make(map[string]Method)} + evalTypes[t] = nt + } + + var et Type + switch t := t.(type) { + case *reflect.BoolType: + et = BoolType + case *reflect.FloatType: + switch t.Kind() { + case reflect.Float32: + et = Float32Type + case reflect.Float64: + et = Float64Type + } + case *reflect.IntType: + switch t.Kind() { + case reflect.Int16: + et = Int16Type + case reflect.Int32: + et = Int32Type + case reflect.Int64: + et = Int64Type + case reflect.Int8: + et = Int8Type + case reflect.Int: + et = IntType + } + case *reflect.UintType: + switch t.Kind() { + case reflect.Uint16: + et = Uint16Type + case reflect.Uint32: + et = Uint32Type + case reflect.Uint64: + et = Uint64Type + case reflect.Uint8: + et = Uint8Type + case reflect.Uint: + et = UintType + case reflect.Uintptr: + et = UintptrType + } + case *reflect.StringType: + et = StringType + case *reflect.ArrayType: + et = NewArrayType(int64(t.Len()), TypeFromNative(t.Elem())) + case *reflect.ChanType: + log.Panicf("%T not implemented", t) + case *reflect.FuncType: + nin := t.NumIn() + // Variadic functions have DotDotDotType at the end + variadic := t.DotDotDot() + if variadic { + nin-- + } + in := make([]Type, nin) + for i := range in { + in[i] = TypeFromNative(t.In(i)) + } + out := make([]Type, t.NumOut()) + for i := range out { + out[i] = TypeFromNative(t.Out(i)) + } + et = NewFuncType(in, variadic, out) + case *reflect.InterfaceType: + log.Panicf("%T not implemented", t) + case *reflect.MapType: + log.Panicf("%T not implemented", t) + case *reflect.PtrType: + et = NewPtrType(TypeFromNative(t.Elem())) + case *reflect.SliceType: + et = NewSliceType(TypeFromNative(t.Elem())) + case *reflect.StructType: + n := t.NumField() + fields := make([]StructField, n) + for i := 0; i < n; i++ { + sf := t.Field(i) + // TODO(austin) What to do about private fields? + fields[i].Name = sf.Name + fields[i].Type = TypeFromNative(sf.Type) + fields[i].Anonymous = sf.Anonymous + } + et = NewStructType(fields) + case *reflect.UnsafePointerType: + log.Panicf("%T not implemented", t) + default: + log.Panicf("unexpected reflect.Type: %T", t) + } + + if nt != nil { + if _, ok := et.(*NamedType); !ok { + nt.Complete(et) + et = nt + } + } + + nativeTypes[et] = t + evalTypes[t] = et + + return et +} + +// TypeOfNative returns the interpreter Type of a regular Go value. +func TypeOfNative(v interface{}) Type { return TypeFromNative(reflect.Typeof(v)) } + +/* + * Function bridging + */ + +type nativeFunc struct { + fn func(*Thread, []Value, []Value) + in, out int +} + +func (f *nativeFunc) NewFrame() *Frame { + vars := make([]Value, f.in+f.out) + return &Frame{nil, vars} +} + +func (f *nativeFunc) Call(t *Thread) { f.fn(t, t.f.Vars[0:f.in], t.f.Vars[f.in:f.in+f.out]) } + +// FuncFromNative creates an interpreter function from a native +// function that takes its in and out arguments as slices of +// interpreter Value's. While somewhat inconvenient, this avoids +// value marshalling. +func FuncFromNative(fn func(*Thread, []Value, []Value), t *FuncType) FuncValue { + return &funcV{&nativeFunc{fn, len(t.In), len(t.Out)}} +} + +// FuncFromNativeTyped is like FuncFromNative, but constructs the +// function type from a function pointer using reflection. Typically, +// the type will be given as a nil pointer to a function with the +// desired signature. +func FuncFromNativeTyped(fn func(*Thread, []Value, []Value), t interface{}) (*FuncType, FuncValue) { + ft := TypeOfNative(t).(*FuncType) + return ft, FuncFromNative(fn, ft) +} diff --git a/libgo/go/exp/eval/compiler.go b/libgo/go/exp/eval/compiler.go new file mode 100644 index 000000000..9d2923bfc --- /dev/null +++ b/libgo/go/exp/eval/compiler.go @@ -0,0 +1,92 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package eval + +import ( + "fmt" + "go/scanner" + "go/token" +) + + +// A compiler captures information used throughout an entire +// compilation. Currently it includes only the error handler. +// +// TODO(austin) This might actually represent package level, in which +// case it should be package compiler. +type compiler struct { + fset *token.FileSet + errors scanner.ErrorHandler + numErrors int + silentErrors int +} + +func (a *compiler) diagAt(pos token.Pos, format string, args ...interface{}) { + a.errors.Error(a.fset.Position(pos), fmt.Sprintf(format, args...)) + a.numErrors++ +} + +func (a *compiler) numError() int { return a.numErrors + a.silentErrors } + +// The universal scope +func newUniverse() *Scope { + sc := &Scope{nil, 0} + sc.block = &block{ + offset: 0, + scope: sc, + global: true, + defs: make(map[string]Def), + } + return sc +} + +var universe *Scope = newUniverse() + + +// TODO(austin) These can all go in stmt.go now +type label struct { + name string + desc string + // The PC goto statements should jump to, or nil if this label + // cannot be goto'd (such as an anonymous for loop label). + gotoPC *uint + // The PC break statements should jump to, or nil if a break + // statement is invalid. + breakPC *uint + // The PC continue statements should jump to, or nil if a + // continue statement is invalid. + continuePC *uint + // The position where this label was resolved. If it has not + // been resolved yet, an invalid position. + resolved token.Pos + // The position where this label was first jumped to. + used token.Pos +} + +// A funcCompiler captures information used throughout the compilation +// of a single function body. +type funcCompiler struct { + *compiler + fnType *FuncType + // Whether the out variables are named. This affects what + // kinds of return statements are legal. + outVarsNamed bool + *codeBuf + flow *flowBuf + labels map[string]*label +} + +// A blockCompiler captures information used throughout the compilation +// of a single block within a function. +type blockCompiler struct { + *funcCompiler + block *block + // The label of this block, used for finding break and + // continue labels. + label *label + // The blockCompiler for the block enclosing this one, or nil + // for a function-level block. + parent *blockCompiler +} diff --git a/libgo/go/exp/eval/eval_test.go b/libgo/go/exp/eval/eval_test.go new file mode 100644 index 000000000..ff28cf1a9 --- /dev/null +++ b/libgo/go/exp/eval/eval_test.go @@ -0,0 +1,259 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package eval + +import ( + "big" + "flag" + "fmt" + "go/token" + "log" + "os" + "reflect" + "regexp" + "testing" +) + +// All tests are done using the same file set. +var fset = token.NewFileSet() + +// Print each statement or expression before parsing it +var noisy = false + +func init() { flag.BoolVar(&noisy, "noisy", false, "chatter during eval tests") } + +/* + * Generic statement/expression test framework + */ + +type test []job + +type job struct { + code string + cerr string + rterr string + val Value + noval bool +} + +func runTests(t *testing.T, baseName string, tests []test) { + for i, test := range tests { + name := fmt.Sprintf("%s[%d]", baseName, i) + test.run(t, name) + } +} + +func (a test) run(t *testing.T, name string) { + w := newTestWorld() + for _, j := range a { + src := j.code + ";" // trailing semicolon to finish statement + if noisy { + println("code:", src) + } + + code, err := w.Compile(fset, src) + if err != nil { + if j.cerr == "" { + t.Errorf("%s: Compile %s: %v", name, src, err) + break + } + if !match(t, err, j.cerr) { + t.Errorf("%s: Compile %s = error %s; want %v", name, src, err, j.cerr) + break + } + continue + } + if j.cerr != "" { + t.Errorf("%s: Compile %s succeeded; want %s", name, src, j.cerr) + break + } + + val, err := code.Run() + if err != nil { + if j.rterr == "" { + t.Errorf("%s: Run %s: %v", name, src, err) + break + } + if !match(t, err, j.rterr) { + t.Errorf("%s: Run %s = error %s; want %v", name, src, err, j.rterr) + break + } + continue + } + if j.rterr != "" { + t.Errorf("%s: Run %s succeeded; want %s", name, src, j.rterr) + break + } + + if !j.noval && !reflect.DeepEqual(val, j.val) { + t.Errorf("%s: Run %s = %T(%v) want %T(%v)", name, src, val, val, j.val, j.val) + } + } +} + +func match(t *testing.T, err os.Error, pat string) bool { + ok, err1 := regexp.MatchString(pat, err.String()) + if err1 != nil { + t.Fatalf("compile regexp %s: %v", pat, err1) + } + return ok +} + + +/* + * Test constructors + */ + +// Expression compile error +func CErr(expr string, cerr string) test { return test([]job{{code: expr, cerr: cerr}}) } + +// Expression runtime error +func RErr(expr string, rterr string) test { return test([]job{{code: expr, rterr: rterr}}) } + +// Expression value +func Val(expr string, val interface{}) test { + return test([]job{{code: expr, val: toValue(val)}}) +} + +// Statement runs without error +func Run(stmts string) test { return test([]job{{code: stmts, noval: true}}) } + +// Two statements without error. +// TODO(rsc): Should be possible with Run but the parser +// won't let us do both top-level and non-top-level statements. +func Run2(stmt1, stmt2 string) test { + return test([]job{{code: stmt1, noval: true}, {code: stmt2, noval: true}}) +} + +// Statement runs and test one expression's value +func Val1(stmts string, expr1 string, val1 interface{}) test { + return test([]job{ + {code: stmts, noval: true}, + {code: expr1, val: toValue(val1)}, + }) +} + +// Statement runs and test two expressions' values +func Val2(stmts string, expr1 string, val1 interface{}, expr2 string, val2 interface{}) test { + return test([]job{ + {code: stmts, noval: true}, + {code: expr1, val: toValue(val1)}, + {code: expr2, val: toValue(val2)}, + }) +} + +/* + * Value constructors + */ + +type vstruct []interface{} + +type varray []interface{} + +type vslice struct { + arr varray + len, cap int +} + +func toValue(val interface{}) Value { + switch val := val.(type) { + case bool: + r := boolV(val) + return &r + case uint8: + r := uint8V(val) + return &r + case uint: + r := uintV(val) + return &r + case int: + r := intV(val) + return &r + case *big.Int: + return &idealIntV{val} + case float64: + r := float64V(val) + return &r + case *big.Rat: + return &idealFloatV{val} + case string: + r := stringV(val) + return &r + case vstruct: + elems := make([]Value, len(val)) + for i, e := range val { + elems[i] = toValue(e) + } + r := structV(elems) + return &r + case varray: + elems := make([]Value, len(val)) + for i, e := range val { + elems[i] = toValue(e) + } + r := arrayV(elems) + return &r + case vslice: + return &sliceV{Slice{toValue(val.arr).(ArrayValue), int64(val.len), int64(val.cap)}} + case Func: + return &funcV{val} + } + log.Panicf("toValue(%T) not implemented", val) + panic("unreachable") +} + +/* + * Default test scope + */ + +type testFunc struct{} + +func (*testFunc) NewFrame() *Frame { return &Frame{nil, make([]Value, 2)} } + +func (*testFunc) Call(t *Thread) { + n := t.f.Vars[0].(IntValue).Get(t) + + res := n + 1 + + t.f.Vars[1].(IntValue).Set(t, res) +} + +type oneTwoFunc struct{} + +func (*oneTwoFunc) NewFrame() *Frame { return &Frame{nil, make([]Value, 2)} } + +func (*oneTwoFunc) Call(t *Thread) { + t.f.Vars[0].(IntValue).Set(t, 1) + t.f.Vars[1].(IntValue).Set(t, 2) +} + +type voidFunc struct{} + +func (*voidFunc) NewFrame() *Frame { return &Frame{nil, []Value{}} } + +func (*voidFunc) Call(t *Thread) {} + +func newTestWorld() *World { + w := NewWorld() + + def := func(name string, t Type, val interface{}) { w.DefineVar(name, t, toValue(val)) } + + w.DefineConst("c", IdealIntType, toValue(big.NewInt(1))) + def("i", IntType, 1) + def("i2", IntType, 2) + def("u", UintType, uint(1)) + def("f", Float64Type, 1.0) + def("s", StringType, "abc") + def("t", NewStructType([]StructField{{"a", IntType, false}}), vstruct{1}) + def("ai", NewArrayType(2, IntType), varray{1, 2}) + def("aai", NewArrayType(2, NewArrayType(2, IntType)), varray{varray{1, 2}, varray{3, 4}}) + def("aai2", NewArrayType(2, NewArrayType(2, IntType)), varray{varray{5, 6}, varray{7, 8}}) + def("fn", NewFuncType([]Type{IntType}, false, []Type{IntType}), &testFunc{}) + def("oneTwo", NewFuncType([]Type{}, false, []Type{IntType, IntType}), &oneTwoFunc{}) + def("void", NewFuncType([]Type{}, false, []Type{}), &voidFunc{}) + def("sli", NewSliceType(IntType), vslice{varray{1, 2, 3}, 2, 3}) + + return w +} diff --git a/libgo/go/exp/eval/expr.go b/libgo/go/exp/eval/expr.go new file mode 100644 index 000000000..e65f47617 --- /dev/null +++ b/libgo/go/exp/eval/expr.go @@ -0,0 +1,2015 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package eval + +import ( + "big" + "fmt" + "go/ast" + "go/token" + "log" + "strconv" + "strings" + "os" +) + +var ( + idealZero = big.NewInt(0) + idealOne = big.NewInt(1) +) + +// An expr is the result of compiling an expression. It stores the +// type of the expression and its evaluator function. +type expr struct { + *exprInfo + t Type + + // Evaluate this node as the given type. + eval interface{} + + // Map index expressions permit special forms of assignment, + // for which we need to know the Map and key. + evalMapValue func(t *Thread) (Map, interface{}) + + // Evaluate to the "address of" this value; that is, the + // settable Value object. nil for expressions whose address + // cannot be taken. + evalAddr func(t *Thread) Value + + // Execute this expression as a statement. Only expressions + // that are valid expression statements should set this. + exec func(t *Thread) + + // If this expression is a type, this is its compiled type. + // This is only permitted in the function position of a call + // expression. In this case, t should be nil. + valType Type + + // A short string describing this expression for error + // messages. + desc string +} + +// exprInfo stores information needed to compile any expression node. +// Each expr also stores its exprInfo so further expressions can be +// compiled from it. +type exprInfo struct { + *compiler + pos token.Pos +} + +func (a *exprInfo) newExpr(t Type, desc string) *expr { + return &expr{exprInfo: a, t: t, desc: desc} +} + +func (a *exprInfo) diag(format string, args ...interface{}) { + a.diagAt(a.pos, format, args...) +} + +func (a *exprInfo) diagOpType(op token.Token, vt Type) { + a.diag("illegal operand type for '%v' operator\n\t%v", op, vt) +} + +func (a *exprInfo) diagOpTypes(op token.Token, lt Type, rt Type) { + a.diag("illegal operand types for '%v' operator\n\t%v\n\t%v", op, lt, rt) +} + +/* + * Common expression manipulations + */ + +// a.convertTo(t) converts the value of the analyzed expression a, +// which must be a constant, ideal number, to a new analyzed +// expression with a constant value of type t. +// +// TODO(austin) Rename to resolveIdeal or something? +func (a *expr) convertTo(t Type) *expr { + if !a.t.isIdeal() { + log.Panicf("attempted to convert from %v, expected ideal", a.t) + } + + var rat *big.Rat + + // XXX(Spec) The spec says "It is erroneous". + // + // It is an error to assign a value with a non-zero fractional + // part to an integer, or if the assignment would overflow or + // underflow, or in general if the value cannot be represented + // by the type of the variable. + switch a.t { + case IdealFloatType: + rat = a.asIdealFloat()() + if t.isInteger() && !rat.IsInt() { + a.diag("constant %v truncated to integer", rat.FloatString(6)) + return nil + } + case IdealIntType: + i := a.asIdealInt()() + rat = new(big.Rat).SetInt(i) + default: + log.Panicf("unexpected ideal type %v", a.t) + } + + // Check bounds + if t, ok := t.lit().(BoundedType); ok { + if rat.Cmp(t.minVal()) < 0 { + a.diag("constant %v underflows %v", rat.FloatString(6), t) + return nil + } + if rat.Cmp(t.maxVal()) > 0 { + a.diag("constant %v overflows %v", rat.FloatString(6), t) + return nil + } + } + + // Convert rat to type t. + res := a.newExpr(t, a.desc) + switch t := t.lit().(type) { + case *uintType: + n, d := rat.Num(), rat.Denom() + f := new(big.Int).Quo(n, d) + f = f.Abs(f) + v := uint64(f.Int64()) + res.eval = func(*Thread) uint64 { return v } + case *intType: + n, d := rat.Num(), rat.Denom() + f := new(big.Int).Quo(n, d) + v := f.Int64() + res.eval = func(*Thread) int64 { return v } + case *idealIntType: + n, d := rat.Num(), rat.Denom() + f := new(big.Int).Quo(n, d) + res.eval = func() *big.Int { return f } + case *floatType: + n, d := rat.Num(), rat.Denom() + v := float64(n.Int64()) / float64(d.Int64()) + res.eval = func(*Thread) float64 { return v } + case *idealFloatType: + res.eval = func() *big.Rat { return rat } + default: + log.Panicf("cannot convert to type %T", t) + } + + return res +} + +// convertToInt converts this expression to an integer, if possible, +// or produces an error if not. This accepts ideal ints, uints, and +// ints. If max is not -1, produces an error if possible if the value +// exceeds max. If negErr is not "", produces an error if possible if +// the value is negative. +func (a *expr) convertToInt(max int64, negErr string, errOp string) *expr { + switch a.t.lit().(type) { + case *idealIntType: + val := a.asIdealInt()() + if negErr != "" && val.Sign() < 0 { + a.diag("negative %s: %s", negErr, val) + return nil + } + bound := max + if negErr == "slice" { + bound++ + } + if max != -1 && val.Cmp(big.NewInt(bound)) >= 0 { + a.diag("index %s exceeds length %d", val, max) + return nil + } + return a.convertTo(IntType) + + case *uintType: + // Convert to int + na := a.newExpr(IntType, a.desc) + af := a.asUint() + na.eval = func(t *Thread) int64 { return int64(af(t)) } + return na + + case *intType: + // Good as is + return a + } + + a.diag("illegal operand type for %s\n\t%v", errOp, a.t) + return nil +} + +// derefArray returns an expression of array type if the given +// expression is a *array type. Otherwise, returns the given +// expression. +func (a *expr) derefArray() *expr { + if pt, ok := a.t.lit().(*PtrType); ok { + if _, ok := pt.Elem.lit().(*ArrayType); ok { + deref := a.compileStarExpr(a) + if deref == nil { + log.Panicf("failed to dereference *array") + } + return deref + } + } + return a +} + +/* + * Assignments + */ + +// An assignCompiler compiles assignment operations. Anything other +// than short declarations should use the compileAssign wrapper. +// +// There are three valid types of assignment: +// 1) T = T +// Assigning a single expression with single-valued type to a +// single-valued type. +// 2) MT = T, T, ... +// Assigning multiple expressions with single-valued types to a +// multi-valued type. +// 3) MT = MT +// Assigning a single expression with multi-valued type to a +// multi-valued type. +type assignCompiler struct { + *compiler + pos token.Pos + // The RHS expressions. This may include nil's for + // expressions that failed to compile. + rs []*expr + // The (possibly unary) MultiType of the RHS. + rmt *MultiType + // Whether this is an unpack assignment (case 3). + isUnpack bool + // Whether map special assignment forms are allowed. + allowMap bool + // Whether this is a "r, ok = a[x]" assignment. + isMapUnpack bool + // The operation name to use in error messages, such as + // "assignment" or "function call". + errOp string + // The name to use for positions in error messages, such as + // "argument". + errPosName string +} + +// Type check the RHS of an assignment, returning a new assignCompiler +// and indicating if the type check succeeded. This always returns an +// assignCompiler with rmt set, but if type checking fails, slots in +// the MultiType may be nil. If rs contains nil's, type checking will +// fail and these expressions given a nil type. +func (a *compiler) checkAssign(pos token.Pos, rs []*expr, errOp, errPosName string) (*assignCompiler, bool) { + c := &assignCompiler{ + compiler: a, + pos: pos, + rs: rs, + errOp: errOp, + errPosName: errPosName, + } + + // Is this an unpack? + if len(rs) == 1 && rs[0] != nil { + if rmt, isUnpack := rs[0].t.(*MultiType); isUnpack { + c.rmt = rmt + c.isUnpack = true + return c, true + } + } + + // Create MultiType for RHS and check that all RHS expressions + // are single-valued. + rts := make([]Type, len(rs)) + ok := true + for i, r := range rs { + if r == nil { + ok = false + continue + } + + if _, isMT := r.t.(*MultiType); isMT { + r.diag("multi-valued expression not allowed in %s", errOp) + ok = false + continue + } + + rts[i] = r.t + } + + c.rmt = NewMultiType(rts) + return c, ok +} + +func (a *assignCompiler) allowMapForms(nls int) { + a.allowMap = true + + // Update unpacking info if this is r, ok = a[x] + if nls == 2 && len(a.rs) == 1 && a.rs[0] != nil && a.rs[0].evalMapValue != nil { + a.isUnpack = true + a.rmt = NewMultiType([]Type{a.rs[0].t, BoolType}) + a.isMapUnpack = true + } +} + +// compile type checks and compiles an assignment operation, returning +// a function that expects an l-value and the frame in which to +// evaluate the RHS expressions. The l-value must have exactly the +// type given by lt. Returns nil if type checking fails. +func (a *assignCompiler) compile(b *block, lt Type) func(Value, *Thread) { + lmt, isMT := lt.(*MultiType) + rmt, isUnpack := a.rmt, a.isUnpack + + // Create unary MultiType for single LHS + if !isMT { + lmt = NewMultiType([]Type{lt}) + } + + // Check that the assignment count matches + lcount := len(lmt.Elems) + rcount := len(rmt.Elems) + if lcount != rcount { + msg := "not enough" + pos := a.pos + if rcount > lcount { + msg = "too many" + if lcount > 0 { + pos = a.rs[lcount-1].pos + } + } + a.diagAt(pos, "%s %ss for %s\n\t%s\n\t%s", msg, a.errPosName, a.errOp, lt, rmt) + return nil + } + + bad := false + + // If this is an unpack, create a temporary to store the + // multi-value and replace the RHS with expressions to pull + // out values from the temporary. Technically, this is only + // necessary when we need to perform assignment conversions. + var effect func(*Thread) + if isUnpack { + // This leaks a slot, but is definitely safe. + temp := b.DefineTemp(a.rmt) + tempIdx := temp.Index + if tempIdx < 0 { + panic(fmt.Sprintln("tempidx", tempIdx)) + } + if a.isMapUnpack { + rf := a.rs[0].evalMapValue + vt := a.rmt.Elems[0] + effect = func(t *Thread) { + m, k := rf(t) + v := m.Elem(t, k) + found := boolV(true) + if v == nil { + found = boolV(false) + v = vt.Zero() + } + t.f.Vars[tempIdx] = multiV([]Value{v, &found}) + } + } else { + rf := a.rs[0].asMulti() + effect = func(t *Thread) { t.f.Vars[tempIdx] = multiV(rf(t)) } + } + orig := a.rs[0] + a.rs = make([]*expr, len(a.rmt.Elems)) + for i, t := range a.rmt.Elems { + if t.isIdeal() { + log.Panicf("Right side of unpack contains ideal: %s", rmt) + } + a.rs[i] = orig.newExpr(t, orig.desc) + index := i + a.rs[i].genValue(func(t *Thread) Value { return t.f.Vars[tempIdx].(multiV)[index] }) + } + } + // Now len(a.rs) == len(a.rmt) and we've reduced any unpacking + // to multi-assignment. + + // TODO(austin) Deal with assignment special cases. + + // Values of any type may always be assigned to variables of + // compatible static type. + for i, lt := range lmt.Elems { + rt := rmt.Elems[i] + + // When [an ideal is] (used in an expression) assigned + // to a variable or typed constant, the destination + // must be able to represent the assigned value. + if rt.isIdeal() { + a.rs[i] = a.rs[i].convertTo(lmt.Elems[i]) + if a.rs[i] == nil { + bad = true + continue + } + rt = a.rs[i].t + } + + // A pointer p to an array can be assigned to a slice + // variable v with compatible element type if the type + // of p or v is unnamed. + if rpt, ok := rt.lit().(*PtrType); ok { + if at, ok := rpt.Elem.lit().(*ArrayType); ok { + if lst, ok := lt.lit().(*SliceType); ok { + if lst.Elem.compat(at.Elem, false) && (rt.lit() == Type(rt) || lt.lit() == Type(lt)) { + rf := a.rs[i].asPtr() + a.rs[i] = a.rs[i].newExpr(lt, a.rs[i].desc) + len := at.Len + a.rs[i].eval = func(t *Thread) Slice { return Slice{rf(t).(ArrayValue), len, len} } + rt = a.rs[i].t + } + } + } + } + + if !lt.compat(rt, false) { + if len(a.rs) == 1 { + a.rs[0].diag("illegal operand types for %s\n\t%v\n\t%v", a.errOp, lt, rt) + } else { + a.rs[i].diag("illegal operand types in %s %d of %s\n\t%v\n\t%v", a.errPosName, i+1, a.errOp, lt, rt) + } + bad = true + } + } + if bad { + return nil + } + + // Compile + if !isMT { + // Case 1 + return genAssign(lt, a.rs[0]) + } + // Case 2 or 3 + as := make([]func(lv Value, t *Thread), len(a.rs)) + for i, r := range a.rs { + as[i] = genAssign(lmt.Elems[i], r) + } + return func(lv Value, t *Thread) { + if effect != nil { + effect(t) + } + lmv := lv.(multiV) + for i, a := range as { + a(lmv[i], t) + } + } +} + +// compileAssign compiles an assignment operation without the full +// generality of an assignCompiler. See assignCompiler for a +// description of the arguments. +func (a *compiler) compileAssign(pos token.Pos, b *block, lt Type, rs []*expr, errOp, errPosName string) func(Value, *Thread) { + ac, ok := a.checkAssign(pos, rs, errOp, errPosName) + if !ok { + return nil + } + return ac.compile(b, lt) +} + +/* + * Expression compiler + */ + +// An exprCompiler stores information used throughout the compilation +// of a single expression. It does not embed funcCompiler because +// expressions can appear at top level. +type exprCompiler struct { + *compiler + // The block this expression is being compiled in. + block *block + // Whether this expression is used in a constant context. + constant bool +} + +// compile compiles an expression AST. callCtx should be true if this +// AST is in the function position of a function call node; it allows +// the returned expression to be a type or a built-in function (which +// otherwise result in errors). +func (a *exprCompiler) compile(x ast.Expr, callCtx bool) *expr { + ei := &exprInfo{a.compiler, x.Pos()} + + switch x := x.(type) { + // Literals + case *ast.BasicLit: + switch x.Kind { + case token.INT: + return ei.compileIntLit(string(x.Value)) + case token.FLOAT: + return ei.compileFloatLit(string(x.Value)) + case token.CHAR: + return ei.compileCharLit(string(x.Value)) + case token.STRING: + return ei.compileStringLit(string(x.Value)) + default: + log.Panicf("unexpected basic literal type %v", x.Kind) + } + + case *ast.CompositeLit: + goto notimpl + + case *ast.FuncLit: + decl := ei.compileFuncType(a.block, x.Type) + if decl == nil { + // TODO(austin) Try compiling the body, + // perhaps with dummy argument definitions + return nil + } + fn := ei.compileFunc(a.block, decl, x.Body) + if fn == nil { + return nil + } + if a.constant { + a.diagAt(x.Pos(), "function literal used in constant expression") + return nil + } + return ei.compileFuncLit(decl, fn) + + // Types + case *ast.ArrayType: + // TODO(austin) Use a multi-type case + goto typeexpr + + case *ast.ChanType: + goto typeexpr + + case *ast.Ellipsis: + goto typeexpr + + case *ast.FuncType: + goto typeexpr + + case *ast.InterfaceType: + goto typeexpr + + case *ast.MapType: + goto typeexpr + + // Remaining expressions + case *ast.BadExpr: + // Error already reported by parser + a.silentErrors++ + return nil + + case *ast.BinaryExpr: + l, r := a.compile(x.X, false), a.compile(x.Y, false) + if l == nil || r == nil { + return nil + } + return ei.compileBinaryExpr(x.Op, l, r) + + case *ast.CallExpr: + l := a.compile(x.Fun, true) + args := make([]*expr, len(x.Args)) + bad := false + for i, arg := range x.Args { + if i == 0 && l != nil && (l.t == Type(makeType) || l.t == Type(newType)) { + argei := &exprInfo{a.compiler, arg.Pos()} + args[i] = argei.exprFromType(a.compileType(a.block, arg)) + } else { + args[i] = a.compile(arg, false) + } + if args[i] == nil { + bad = true + } + } + if bad || l == nil { + return nil + } + if a.constant { + a.diagAt(x.Pos(), "function call in constant context") + return nil + } + + if l.valType != nil { + a.diagAt(x.Pos(), "type conversions not implemented") + return nil + } else if ft, ok := l.t.(*FuncType); ok && ft.builtin != "" { + return ei.compileBuiltinCallExpr(a.block, ft, args) + } else { + return ei.compileCallExpr(a.block, l, args) + } + + case *ast.Ident: + return ei.compileIdent(a.block, a.constant, callCtx, x.Name) + + case *ast.IndexExpr: + l, r := a.compile(x.X, false), a.compile(x.Index, false) + if l == nil || r == nil { + return nil + } + return ei.compileIndexExpr(l, r) + + case *ast.SliceExpr: + var lo, hi *expr + arr := a.compile(x.X, false) + if x.Low == nil { + // beginning was omitted, so we need to provide it + ei := &exprInfo{a.compiler, x.Pos()} + lo = ei.compileIntLit("0") + } else { + lo = a.compile(x.Low, false) + } + if x.High == nil { + // End was omitted, so we need to compute len(x.X) + ei := &exprInfo{a.compiler, x.Pos()} + hi = ei.compileBuiltinCallExpr(a.block, lenType, []*expr{arr}) + } else { + hi = a.compile(x.High, false) + } + if arr == nil || lo == nil || hi == nil { + return nil + } + return ei.compileSliceExpr(arr, lo, hi) + + case *ast.KeyValueExpr: + goto notimpl + + case *ast.ParenExpr: + return a.compile(x.X, callCtx) + + case *ast.SelectorExpr: + v := a.compile(x.X, false) + if v == nil { + return nil + } + return ei.compileSelectorExpr(v, x.Sel.Name) + + case *ast.StarExpr: + // We pass down our call context because this could be + // a pointer type (and thus a type conversion) + v := a.compile(x.X, callCtx) + if v == nil { + return nil + } + if v.valType != nil { + // Turns out this was a pointer type, not a dereference + return ei.exprFromType(NewPtrType(v.valType)) + } + return ei.compileStarExpr(v) + + case *ast.StructType: + goto notimpl + + case *ast.TypeAssertExpr: + goto notimpl + + case *ast.UnaryExpr: + v := a.compile(x.X, false) + if v == nil { + return nil + } + return ei.compileUnaryExpr(x.Op, v) + } + log.Panicf("unexpected ast node type %T", x) + panic("unreachable") + +typeexpr: + if !callCtx { + a.diagAt(x.Pos(), "type used as expression") + return nil + } + return ei.exprFromType(a.compileType(a.block, x)) + +notimpl: + a.diagAt(x.Pos(), "%T expression node not implemented", x) + return nil +} + +func (a *exprInfo) exprFromType(t Type) *expr { + if t == nil { + return nil + } + expr := a.newExpr(nil, "type") + expr.valType = t + return expr +} + +func (a *exprInfo) compileIdent(b *block, constant bool, callCtx bool, name string) *expr { + bl, level, def := b.Lookup(name) + if def == nil { + a.diag("%s: undefined", name) + return nil + } + switch def := def.(type) { + case *Constant: + expr := a.newExpr(def.Type, "constant") + if ft, ok := def.Type.(*FuncType); ok && ft.builtin != "" { + // XXX(Spec) I don't think anything says that + // built-in functions can't be used as values. + if !callCtx { + a.diag("built-in function %s cannot be used as a value", ft.builtin) + return nil + } + // Otherwise, we leave the evaluators empty + // because this is handled specially + } else { + expr.genConstant(def.Value) + } + return expr + case *Variable: + if constant { + a.diag("variable %s used in constant expression", name) + return nil + } + if bl.global { + return a.compileGlobalVariable(def) + } + return a.compileVariable(level, def) + case Type: + if callCtx { + return a.exprFromType(def) + } + a.diag("type %v used as expression", name) + return nil + } + log.Panicf("name %s has unknown type %T", name, def) + panic("unreachable") +} + +func (a *exprInfo) compileVariable(level int, v *Variable) *expr { + if v.Type == nil { + // Placeholder definition from an earlier error + a.silentErrors++ + return nil + } + expr := a.newExpr(v.Type, "variable") + expr.genIdentOp(level, v.Index) + return expr +} + +func (a *exprInfo) compileGlobalVariable(v *Variable) *expr { + if v.Type == nil { + // Placeholder definition from an earlier error + a.silentErrors++ + return nil + } + if v.Init == nil { + v.Init = v.Type.Zero() + } + expr := a.newExpr(v.Type, "variable") + val := v.Init + expr.genValue(func(t *Thread) Value { return val }) + return expr +} + +func (a *exprInfo) compileIdealInt(i *big.Int, desc string) *expr { + expr := a.newExpr(IdealIntType, desc) + expr.eval = func() *big.Int { return i } + return expr +} + +func (a *exprInfo) compileIntLit(lit string) *expr { + i, _ := new(big.Int).SetString(lit, 0) + return a.compileIdealInt(i, "integer literal") +} + +func (a *exprInfo) compileCharLit(lit string) *expr { + if lit[0] != '\'' { + // Caught by parser + a.silentErrors++ + return nil + } + v, _, tail, err := strconv.UnquoteChar(lit[1:], '\'') + if err != nil || tail != "'" { + // Caught by parser + a.silentErrors++ + return nil + } + return a.compileIdealInt(big.NewInt(int64(v)), "character literal") +} + +func (a *exprInfo) compileFloatLit(lit string) *expr { + f, ok := new(big.Rat).SetString(lit) + if !ok { + log.Panicf("malformed float literal %s at %v passed parser", lit, a.pos) + } + expr := a.newExpr(IdealFloatType, "float literal") + expr.eval = func() *big.Rat { return f } + return expr +} + +func (a *exprInfo) compileString(s string) *expr { + // Ideal strings don't have a named type but they are + // compatible with type string. + + // TODO(austin) Use unnamed string type. + expr := a.newExpr(StringType, "string literal") + expr.eval = func(*Thread) string { return s } + return expr +} + +func (a *exprInfo) compileStringLit(lit string) *expr { + s, err := strconv.Unquote(lit) + if err != nil { + a.diag("illegal string literal, %v", err) + return nil + } + return a.compileString(s) +} + +func (a *exprInfo) compileStringList(list []*expr) *expr { + ss := make([]string, len(list)) + for i, s := range list { + ss[i] = s.asString()(nil) + } + return a.compileString(strings.Join(ss, "")) +} + +func (a *exprInfo) compileFuncLit(decl *FuncDecl, fn func(*Thread) Func) *expr { + expr := a.newExpr(decl.Type, "function literal") + expr.eval = fn + return expr +} + +func (a *exprInfo) compileSelectorExpr(v *expr, name string) *expr { + // mark marks a field that matches the selector name. It + // tracks the best depth found so far and whether more than + // one field has been found at that depth. + bestDepth := -1 + ambig := false + amberr := "" + mark := func(depth int, pathName string) { + switch { + case bestDepth == -1 || depth < bestDepth: + bestDepth = depth + ambig = false + amberr = "" + + case depth == bestDepth: + ambig = true + + default: + log.Panicf("Marked field at depth %d, but already found one at depth %d", depth, bestDepth) + } + amberr += "\n\t" + pathName[1:] + } + + visited := make(map[Type]bool) + + // find recursively searches for the named field, starting at + // type t. If it finds the named field, it returns a function + // which takes an expr that represents a value of type 't' and + // returns an expr that retrieves the named field. We delay + // expr construction to avoid producing lots of useless expr's + // as we search. + // + // TODO(austin) Now that the expression compiler works on + // semantic values instead of AST's, there should be a much + // better way of doing this. + var find func(Type, int, string) func(*expr) *expr + find = func(t Type, depth int, pathName string) func(*expr) *expr { + // Don't bother looking if we've found something shallower + if bestDepth != -1 && bestDepth < depth { + return nil + } + + // Don't check the same type twice and avoid loops + if visited[t] { + return nil + } + visited[t] = true + + // Implicit dereference + deref := false + if ti, ok := t.(*PtrType); ok { + deref = true + t = ti.Elem + } + + // If it's a named type, look for methods + if ti, ok := t.(*NamedType); ok { + _, ok := ti.methods[name] + if ok { + mark(depth, pathName+"."+name) + log.Panic("Methods not implemented") + } + t = ti.Def + } + + // If it's a struct type, check fields and embedded types + var builder func(*expr) *expr + if t, ok := t.(*StructType); ok { + for i, f := range t.Elems { + var sub func(*expr) *expr + switch { + case f.Name == name: + mark(depth, pathName+"."+name) + sub = func(e *expr) *expr { return e } + + case f.Anonymous: + sub = find(f.Type, depth+1, pathName+"."+f.Name) + if sub == nil { + continue + } + + default: + continue + } + + // We found something. Create a + // builder for accessing this field. + ft := f.Type + index := i + builder = func(parent *expr) *expr { + if deref { + parent = a.compileStarExpr(parent) + } + expr := a.newExpr(ft, "selector expression") + pf := parent.asStruct() + evalAddr := func(t *Thread) Value { return pf(t).Field(t, index) } + expr.genValue(evalAddr) + return sub(expr) + } + } + } + + return builder + } + + builder := find(v.t, 0, "") + if builder == nil { + a.diag("type %v has no field or method %s", v.t, name) + return nil + } + if ambig { + a.diag("field %s is ambiguous in type %v%s", name, v.t, amberr) + return nil + } + + return builder(v) +} + +func (a *exprInfo) compileSliceExpr(arr, lo, hi *expr) *expr { + // Type check object + arr = arr.derefArray() + + var at Type + var maxIndex int64 = -1 + + switch lt := arr.t.lit().(type) { + case *ArrayType: + at = NewSliceType(lt.Elem) + maxIndex = lt.Len + + case *SliceType: + at = lt + + case *stringType: + at = lt + + default: + a.diag("cannot slice %v", arr.t) + return nil + } + + // Type check index and convert to int + // XXX(Spec) It's unclear if ideal floats with no + // fractional part are allowed here. 6g allows it. I + // believe that's wrong. + lo = lo.convertToInt(maxIndex, "slice", "slice") + hi = hi.convertToInt(maxIndex, "slice", "slice") + if lo == nil || hi == nil { + return nil + } + + expr := a.newExpr(at, "slice expression") + + // Compile + lof := lo.asInt() + hif := hi.asInt() + switch lt := arr.t.lit().(type) { + case *ArrayType: + arrf := arr.asArray() + bound := lt.Len + expr.eval = func(t *Thread) Slice { + arr, lo, hi := arrf(t), lof(t), hif(t) + if lo > hi || hi > bound || lo < 0 { + t.Abort(SliceError{lo, hi, bound}) + } + return Slice{arr.Sub(lo, bound-lo), hi - lo, bound - lo} + } + + case *SliceType: + arrf := arr.asSlice() + expr.eval = func(t *Thread) Slice { + arr, lo, hi := arrf(t), lof(t), hif(t) + if lo > hi || hi > arr.Cap || lo < 0 { + t.Abort(SliceError{lo, hi, arr.Cap}) + } + return Slice{arr.Base.Sub(lo, arr.Cap-lo), hi - lo, arr.Cap - lo} + } + + case *stringType: + arrf := arr.asString() + // TODO(austin) This pulls over the whole string in a + // remote setting, instead of creating a substring backed + // by remote memory. + expr.eval = func(t *Thread) string { + arr, lo, hi := arrf(t), lof(t), hif(t) + if lo > hi || hi > int64(len(arr)) || lo < 0 { + t.Abort(SliceError{lo, hi, int64(len(arr))}) + } + return arr[lo:hi] + } + + default: + log.Panicf("unexpected left operand type %T", arr.t.lit()) + } + + return expr +} + +func (a *exprInfo) compileIndexExpr(l, r *expr) *expr { + // Type check object + l = l.derefArray() + + var at Type + intIndex := false + var maxIndex int64 = -1 + + switch lt := l.t.lit().(type) { + case *ArrayType: + at = lt.Elem + intIndex = true + maxIndex = lt.Len + + case *SliceType: + at = lt.Elem + intIndex = true + + case *stringType: + at = Uint8Type + intIndex = true + + case *MapType: + at = lt.Elem + if r.t.isIdeal() { + r = r.convertTo(lt.Key) + if r == nil { + return nil + } + } + if !lt.Key.compat(r.t, false) { + a.diag("cannot use %s as index into %s", r.t, lt) + return nil + } + + default: + a.diag("cannot index into %v", l.t) + return nil + } + + // Type check index and convert to int if necessary + if intIndex { + // XXX(Spec) It's unclear if ideal floats with no + // fractional part are allowed here. 6g allows it. I + // believe that's wrong. + r = r.convertToInt(maxIndex, "index", "index") + if r == nil { + return nil + } + } + + expr := a.newExpr(at, "index expression") + + // Compile + switch lt := l.t.lit().(type) { + case *ArrayType: + lf := l.asArray() + rf := r.asInt() + bound := lt.Len + expr.genValue(func(t *Thread) Value { + l, r := lf(t), rf(t) + if r < 0 || r >= bound { + t.Abort(IndexError{r, bound}) + } + return l.Elem(t, r) + }) + + case *SliceType: + lf := l.asSlice() + rf := r.asInt() + expr.genValue(func(t *Thread) Value { + l, r := lf(t), rf(t) + if l.Base == nil { + t.Abort(NilPointerError{}) + } + if r < 0 || r >= l.Len { + t.Abort(IndexError{r, l.Len}) + } + return l.Base.Elem(t, r) + }) + + case *stringType: + lf := l.asString() + rf := r.asInt() + // TODO(austin) This pulls over the whole string in a + // remote setting, instead of just the one character. + expr.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + if r < 0 || r >= int64(len(l)) { + t.Abort(IndexError{r, int64(len(l))}) + } + return uint64(l[r]) + } + + case *MapType: + lf := l.asMap() + rf := r.asInterface() + expr.genValue(func(t *Thread) Value { + m := lf(t) + k := rf(t) + if m == nil { + t.Abort(NilPointerError{}) + } + e := m.Elem(t, k) + if e == nil { + t.Abort(KeyError{k}) + } + return e + }) + // genValue makes things addressable, but map values + // aren't addressable. + expr.evalAddr = nil + expr.evalMapValue = func(t *Thread) (Map, interface{}) { + // TODO(austin) Key check? nil check? + return lf(t), rf(t) + } + + default: + log.Panicf("unexpected left operand type %T", l.t.lit()) + } + + return expr +} + +func (a *exprInfo) compileCallExpr(b *block, l *expr, as []*expr) *expr { + // TODO(austin) Variadic functions. + + // Type check + + // XXX(Spec) Calling a named function type is okay. I really + // think there needs to be a general discussion of named + // types. A named type creates a new, distinct type, but the + // type of that type is still whatever it's defined to. Thus, + // in "type Foo int", Foo is still an integer type and in + // "type Foo func()", Foo is a function type. + lt, ok := l.t.lit().(*FuncType) + if !ok { + a.diag("cannot call non-function type %v", l.t) + return nil + } + + // The arguments must be single-valued expressions assignment + // compatible with the parameters of F. + // + // XXX(Spec) The spec is wrong. It can also be a single + // multi-valued expression. + nin := len(lt.In) + assign := a.compileAssign(a.pos, b, NewMultiType(lt.In), as, "function call", "argument") + if assign == nil { + return nil + } + + var t Type + nout := len(lt.Out) + switch nout { + case 0: + t = EmptyType + case 1: + t = lt.Out[0] + default: + t = NewMultiType(lt.Out) + } + expr := a.newExpr(t, "function call") + + // Gather argument and out types to initialize frame variables + vts := make([]Type, nin+nout) + copy(vts, lt.In) + copy(vts[nin:], lt.Out) + + // Compile + lf := l.asFunc() + call := func(t *Thread) []Value { + fun := lf(t) + fr := fun.NewFrame() + for i, t := range vts { + fr.Vars[i] = t.Zero() + } + assign(multiV(fr.Vars[0:nin]), t) + oldf := t.f + t.f = fr + fun.Call(t) + t.f = oldf + return fr.Vars[nin : nin+nout] + } + expr.genFuncCall(call) + + return expr +} + +func (a *exprInfo) compileBuiltinCallExpr(b *block, ft *FuncType, as []*expr) *expr { + checkCount := func(min, max int) bool { + if len(as) < min { + a.diag("not enough arguments to %s", ft.builtin) + return false + } else if len(as) > max { + a.diag("too many arguments to %s", ft.builtin) + return false + } + return true + } + + switch ft { + case capType: + if !checkCount(1, 1) { + return nil + } + arg := as[0].derefArray() + expr := a.newExpr(IntType, "function call") + switch t := arg.t.lit().(type) { + case *ArrayType: + // TODO(austin) It would be nice if this could + // be a constant int. + v := t.Len + expr.eval = func(t *Thread) int64 { return v } + + case *SliceType: + vf := arg.asSlice() + expr.eval = func(t *Thread) int64 { return vf(t).Cap } + + //case *ChanType: + + default: + a.diag("illegal argument type for cap function\n\t%v", arg.t) + return nil + } + return expr + + case copyType: + if !checkCount(2, 2) { + return nil + } + src := as[1] + dst := as[0] + if src.t != dst.t { + a.diag("arguments to built-in function 'copy' must have same type\nsrc: %s\ndst: %s\n", src.t, dst.t) + return nil + } + if _, ok := src.t.lit().(*SliceType); !ok { + a.diag("src argument to 'copy' must be a slice (got: %s)", src.t) + return nil + } + if _, ok := dst.t.lit().(*SliceType); !ok { + a.diag("dst argument to 'copy' must be a slice (got: %s)", dst.t) + return nil + } + expr := a.newExpr(IntType, "function call") + srcf := src.asSlice() + dstf := dst.asSlice() + expr.eval = func(t *Thread) int64 { + src, dst := srcf(t), dstf(t) + nelems := src.Len + if nelems > dst.Len { + nelems = dst.Len + } + dst.Base.Sub(0, nelems).Assign(t, src.Base.Sub(0, nelems)) + return nelems + } + return expr + + case lenType: + if !checkCount(1, 1) { + return nil + } + arg := as[0].derefArray() + expr := a.newExpr(IntType, "function call") + switch t := arg.t.lit().(type) { + case *stringType: + vf := arg.asString() + expr.eval = func(t *Thread) int64 { return int64(len(vf(t))) } + + case *ArrayType: + // TODO(austin) It would be nice if this could + // be a constant int. + v := t.Len + expr.eval = func(t *Thread) int64 { return v } + + case *SliceType: + vf := arg.asSlice() + expr.eval = func(t *Thread) int64 { return vf(t).Len } + + case *MapType: + vf := arg.asMap() + expr.eval = func(t *Thread) int64 { + // XXX(Spec) What's the len of an + // uninitialized map? + m := vf(t) + if m == nil { + return 0 + } + return m.Len(t) + } + + //case *ChanType: + + default: + a.diag("illegal argument type for len function\n\t%v", arg.t) + return nil + } + return expr + + case makeType: + if !checkCount(1, 3) { + return nil + } + // XXX(Spec) What are the types of the + // arguments? Do they have to be ints? 6g + // accepts any integral type. + var lenexpr, capexpr *expr + var lenf, capf func(*Thread) int64 + if len(as) > 1 { + lenexpr = as[1].convertToInt(-1, "length", "make function") + if lenexpr == nil { + return nil + } + lenf = lenexpr.asInt() + } + if len(as) > 2 { + capexpr = as[2].convertToInt(-1, "capacity", "make function") + if capexpr == nil { + return nil + } + capf = capexpr.asInt() + } + + switch t := as[0].valType.lit().(type) { + case *SliceType: + // A new, initialized slice value for a given + // element type T is made using the built-in + // function make, which takes a slice type and + // parameters specifying the length and + // optionally the capacity. + if !checkCount(2, 3) { + return nil + } + et := t.Elem + expr := a.newExpr(t, "function call") + expr.eval = func(t *Thread) Slice { + l := lenf(t) + // XXX(Spec) What if len or cap is + // negative? The runtime panics. + if l < 0 { + t.Abort(NegativeLengthError{l}) + } + c := l + if capf != nil { + c = capf(t) + if c < 0 { + t.Abort(NegativeCapacityError{c}) + } + // XXX(Spec) What happens if + // len > cap? The runtime + // sets cap to len. + if l > c { + c = l + } + } + base := arrayV(make([]Value, c)) + for i := int64(0); i < c; i++ { + base[i] = et.Zero() + } + return Slice{&base, l, c} + } + return expr + + case *MapType: + // A new, empty map value is made using the + // built-in function make, which takes the map + // type and an optional capacity hint as + // arguments. + if !checkCount(1, 2) { + return nil + } + expr := a.newExpr(t, "function call") + expr.eval = func(t *Thread) Map { + if lenf == nil { + return make(evalMap) + } + l := lenf(t) + return make(evalMap, l) + } + return expr + + //case *ChanType: + + default: + a.diag("illegal argument type for make function\n\t%v", as[0].valType) + return nil + } + + case closeType, closedType: + a.diag("built-in function %s not implemented", ft.builtin) + return nil + + case newType: + if !checkCount(1, 1) { + return nil + } + + t := as[0].valType + expr := a.newExpr(NewPtrType(t), "new") + expr.eval = func(*Thread) Value { return t.Zero() } + return expr + + case panicType, printType, printlnType: + evals := make([]func(*Thread) interface{}, len(as)) + for i, x := range as { + evals[i] = x.asInterface() + } + spaces := ft == printlnType + newline := ft != printType + printer := func(t *Thread) { + for i, eval := range evals { + if i > 0 && spaces { + print(" ") + } + v := eval(t) + type stringer interface { + String() string + } + switch v1 := v.(type) { + case bool: + print(v1) + case uint64: + print(v1) + case int64: + print(v1) + case float64: + print(v1) + case string: + print(v1) + case stringer: + print(v1.String()) + default: + print("???") + } + } + if newline { + print("\n") + } + } + expr := a.newExpr(EmptyType, "print") + expr.exec = printer + if ft == panicType { + expr.exec = func(t *Thread) { + printer(t) + t.Abort(os.NewError("panic")) + } + } + return expr + } + + log.Panicf("unexpected built-in function '%s'", ft.builtin) + panic("unreachable") +} + +func (a *exprInfo) compileStarExpr(v *expr) *expr { + switch vt := v.t.lit().(type) { + case *PtrType: + expr := a.newExpr(vt.Elem, "indirect expression") + vf := v.asPtr() + expr.genValue(func(t *Thread) Value { + v := vf(t) + if v == nil { + t.Abort(NilPointerError{}) + } + return v + }) + return expr + } + + a.diagOpType(token.MUL, v.t) + return nil +} + +var unaryOpDescs = make(map[token.Token]string) + +func (a *exprInfo) compileUnaryExpr(op token.Token, v *expr) *expr { + // Type check + var t Type + switch op { + case token.ADD, token.SUB: + if !v.t.isInteger() && !v.t.isFloat() { + a.diagOpType(op, v.t) + return nil + } + t = v.t + + case token.NOT: + if !v.t.isBoolean() { + a.diagOpType(op, v.t) + return nil + } + t = BoolType + + case token.XOR: + if !v.t.isInteger() { + a.diagOpType(op, v.t) + return nil + } + t = v.t + + case token.AND: + // The unary prefix address-of operator & generates + // the address of its operand, which must be a + // variable, pointer indirection, field selector, or + // array or slice indexing operation. + if v.evalAddr == nil { + a.diag("cannot take the address of %s", v.desc) + return nil + } + + // TODO(austin) Implement "It is illegal to take the + // address of a function result variable" once I have + // function result variables. + + t = NewPtrType(v.t) + + case token.ARROW: + log.Panicf("Unary op %v not implemented", op) + + default: + log.Panicf("unknown unary operator %v", op) + } + + desc, ok := unaryOpDescs[op] + if !ok { + desc = "unary " + op.String() + " expression" + unaryOpDescs[op] = desc + } + + // Compile + expr := a.newExpr(t, desc) + switch op { + case token.ADD: + // Just compile it out + expr = v + expr.desc = desc + + case token.SUB: + expr.genUnaryOpNeg(v) + + case token.NOT: + expr.genUnaryOpNot(v) + + case token.XOR: + expr.genUnaryOpXor(v) + + case token.AND: + vf := v.evalAddr + expr.eval = func(t *Thread) Value { return vf(t) } + + default: + log.Panicf("Compilation of unary op %v not implemented", op) + } + + return expr +} + +var binOpDescs = make(map[token.Token]string) + +func (a *exprInfo) compileBinaryExpr(op token.Token, l, r *expr) *expr { + // Save the original types of l.t and r.t for error messages. + origlt := l.t + origrt := r.t + + // XXX(Spec) What is the exact definition of a "named type"? + + // XXX(Spec) Arithmetic operators: "Integer types" apparently + // means all types compatible with basic integer types, though + // this is never explained. Likewise for float types, etc. + // This relates to the missing explanation of named types. + + // XXX(Spec) Operators: "If both operands are ideal numbers, + // the conversion is to ideal floats if one of the operands is + // an ideal float (relevant for / and %)." How is that + // relevant only for / and %? If I add an ideal int and an + // ideal float, I get an ideal float. + + if op != token.SHL && op != token.SHR { + // Except in shift expressions, if one operand has + // numeric type and the other operand is an ideal + // number, the ideal number is converted to match the + // type of the other operand. + if (l.t.isInteger() || l.t.isFloat()) && !l.t.isIdeal() && r.t.isIdeal() { + r = r.convertTo(l.t) + } else if (r.t.isInteger() || r.t.isFloat()) && !r.t.isIdeal() && l.t.isIdeal() { + l = l.convertTo(r.t) + } + if l == nil || r == nil { + return nil + } + + // Except in shift expressions, if both operands are + // ideal numbers and one is an ideal float, the other + // is converted to ideal float. + if l.t.isIdeal() && r.t.isIdeal() { + if l.t.isInteger() && r.t.isFloat() { + l = l.convertTo(r.t) + } else if l.t.isFloat() && r.t.isInteger() { + r = r.convertTo(l.t) + } + if l == nil || r == nil { + return nil + } + } + } + + // Useful type predicates + // TODO(austin) CL 33668 mandates identical types except for comparisons. + compat := func() bool { return l.t.compat(r.t, false) } + integers := func() bool { return l.t.isInteger() && r.t.isInteger() } + floats := func() bool { return l.t.isFloat() && r.t.isFloat() } + strings := func() bool { + // TODO(austin) Deal with named types + return l.t == StringType && r.t == StringType + } + booleans := func() bool { return l.t.isBoolean() && r.t.isBoolean() } + + // Type check + var t Type + switch op { + case token.ADD: + if !compat() || (!integers() && !floats() && !strings()) { + a.diagOpTypes(op, origlt, origrt) + return nil + } + t = l.t + + case token.SUB, token.MUL, token.QUO: + if !compat() || (!integers() && !floats()) { + a.diagOpTypes(op, origlt, origrt) + return nil + } + t = l.t + + case token.REM, token.AND, token.OR, token.XOR, token.AND_NOT: + if !compat() || !integers() { + a.diagOpTypes(op, origlt, origrt) + return nil + } + t = l.t + + case token.SHL, token.SHR: + // XXX(Spec) Is it okay for the right operand to be an + // ideal float with no fractional part? "The right + // operand in a shift operation must be always be of + // unsigned integer type or an ideal number that can + // be safely converted into an unsigned integer type + // (§Arithmetic operators)" suggests so and 6g agrees. + + if !l.t.isInteger() || !(r.t.isInteger() || r.t.isIdeal()) { + a.diagOpTypes(op, origlt, origrt) + return nil + } + + // The right operand in a shift operation must be + // always be of unsigned integer type or an ideal + // number that can be safely converted into an + // unsigned integer type. + if r.t.isIdeal() { + r2 := r.convertTo(UintType) + if r2 == nil { + return nil + } + + // If the left operand is not ideal, convert + // the right to not ideal. + if !l.t.isIdeal() { + r = r2 + } + + // If both are ideal, but the right side isn't + // an ideal int, convert it to simplify things. + if l.t.isIdeal() && !r.t.isInteger() { + r = r.convertTo(IdealIntType) + if r == nil { + log.Panicf("conversion to uintType succeeded, but conversion to idealIntType failed") + } + } + } else if _, ok := r.t.lit().(*uintType); !ok { + a.diag("right operand of shift must be unsigned") + return nil + } + + if l.t.isIdeal() && !r.t.isIdeal() { + // XXX(Spec) What is the meaning of "ideal >> + // non-ideal"? Russ says the ideal should be + // converted to an int. 6g propagates the + // type down from assignments as a hint. + + l = l.convertTo(IntType) + if l == nil { + return nil + } + } + + // At this point, we should have one of three cases: + // 1) uint SHIFT uint + // 2) int SHIFT uint + // 3) ideal int SHIFT ideal int + + t = l.t + + case token.LOR, token.LAND: + if !booleans() { + return nil + } + // XXX(Spec) There's no mention of *which* boolean + // type the logical operators return. From poking at + // 6g, it appears to be the named boolean type, NOT + // the type of the left operand, and NOT an unnamed + // boolean type. + + t = BoolType + + case token.ARROW: + // The operands in channel sends differ in type: one + // is always a channel and the other is a variable or + // value of the channel's element type. + log.Panic("Binary op <- not implemented") + t = BoolType + + case token.LSS, token.GTR, token.LEQ, token.GEQ: + // XXX(Spec) It's really unclear what types which + // comparison operators apply to. I feel like the + // text is trying to paint a Venn diagram for me, + // which it's really pretty simple: <, <=, >, >= apply + // only to numeric types and strings. == and != apply + // to everything except arrays and structs, and there + // are some restrictions on when it applies to slices. + + if !compat() || (!integers() && !floats() && !strings()) { + a.diagOpTypes(op, origlt, origrt) + return nil + } + t = BoolType + + case token.EQL, token.NEQ: + // XXX(Spec) The rules for type checking comparison + // operators are spread across three places that all + // partially overlap with each other: the Comparison + // Compatibility section, the Operators section, and + // the Comparison Operators section. The Operators + // section should just say that operators require + // identical types (as it does currently) except that + // there a few special cases for comparison, which are + // described in section X. Currently it includes just + // one of the four special cases. The Comparison + // Compatibility section and the Comparison Operators + // section should either be merged, or at least the + // Comparison Compatibility section should be + // exclusively about type checking and the Comparison + // Operators section should be exclusively about + // semantics. + + // XXX(Spec) Comparison operators: "All comparison + // operators apply to basic types except bools." This + // is very difficult to parse. It's explained much + // better in the Comparison Compatibility section. + + // XXX(Spec) Comparison compatibility: "Function + // values are equal if they refer to the same + // function." is rather vague. It should probably be + // similar to the way the rule for map values is + // written: Function values are equal if they were + // created by the same execution of a function literal + // or refer to the same function declaration. This is + // *almost* but not quite waht 6g implements. If a + // function literals does not capture any variables, + // then multiple executions of it will result in the + // same closure. Russ says he'll change that. + + // TODO(austin) Deal with remaining special cases + + if !compat() { + a.diagOpTypes(op, origlt, origrt) + return nil + } + // Arrays and structs may not be compared to anything. + switch l.t.(type) { + case *ArrayType, *StructType: + a.diagOpTypes(op, origlt, origrt) + return nil + } + t = BoolType + + default: + log.Panicf("unknown binary operator %v", op) + } + + desc, ok := binOpDescs[op] + if !ok { + desc = op.String() + " expression" + binOpDescs[op] = desc + } + + // Check for ideal divide by zero + switch op { + case token.QUO, token.REM: + if r.t.isIdeal() { + if (r.t.isInteger() && r.asIdealInt()().Sign() == 0) || + (r.t.isFloat() && r.asIdealFloat()().Sign() == 0) { + a.diag("divide by zero") + return nil + } + } + } + + // Compile + expr := a.newExpr(t, desc) + switch op { + case token.ADD: + expr.genBinOpAdd(l, r) + + case token.SUB: + expr.genBinOpSub(l, r) + + case token.MUL: + expr.genBinOpMul(l, r) + + case token.QUO: + expr.genBinOpQuo(l, r) + + case token.REM: + expr.genBinOpRem(l, r) + + case token.AND: + expr.genBinOpAnd(l, r) + + case token.OR: + expr.genBinOpOr(l, r) + + case token.XOR: + expr.genBinOpXor(l, r) + + case token.AND_NOT: + expr.genBinOpAndNot(l, r) + + case token.SHL: + if l.t.isIdeal() { + lv := l.asIdealInt()() + rv := r.asIdealInt()() + const maxShift = 99999 + if rv.Cmp(big.NewInt(maxShift)) > 0 { + a.diag("left shift by %v; exceeds implementation limit of %v", rv, maxShift) + expr.t = nil + return nil + } + val := new(big.Int).Lsh(lv, uint(rv.Int64())) + expr.eval = func() *big.Int { return val } + } else { + expr.genBinOpShl(l, r) + } + + case token.SHR: + if l.t.isIdeal() { + lv := l.asIdealInt()() + rv := r.asIdealInt()() + val := new(big.Int).Rsh(lv, uint(rv.Int64())) + expr.eval = func() *big.Int { return val } + } else { + expr.genBinOpShr(l, r) + } + + case token.LSS: + expr.genBinOpLss(l, r) + + case token.GTR: + expr.genBinOpGtr(l, r) + + case token.LEQ: + expr.genBinOpLeq(l, r) + + case token.GEQ: + expr.genBinOpGeq(l, r) + + case token.EQL: + expr.genBinOpEql(l, r) + + case token.NEQ: + expr.genBinOpNeq(l, r) + + case token.LAND: + expr.genBinOpLogAnd(l, r) + + case token.LOR: + expr.genBinOpLogOr(l, r) + + default: + log.Panicf("Compilation of binary op %v not implemented", op) + } + + return expr +} + +// TODO(austin) This is a hack to eliminate a circular dependency +// between type.go and expr.go +func (a *compiler) compileArrayLen(b *block, expr ast.Expr) (int64, bool) { + lenExpr := a.compileExpr(b, true, expr) + if lenExpr == nil { + return 0, false + } + + // XXX(Spec) Are ideal floats with no fractional part okay? + if lenExpr.t.isIdeal() { + lenExpr = lenExpr.convertTo(IntType) + if lenExpr == nil { + return 0, false + } + } + + if !lenExpr.t.isInteger() { + a.diagAt(expr.Pos(), "array size must be an integer") + return 0, false + } + + switch lenExpr.t.lit().(type) { + case *intType: + return lenExpr.asInt()(nil), true + case *uintType: + return int64(lenExpr.asUint()(nil)), true + } + log.Panicf("unexpected integer type %T", lenExpr.t) + return 0, false +} + +func (a *compiler) compileExpr(b *block, constant bool, expr ast.Expr) *expr { + ec := &exprCompiler{a, b, constant} + nerr := a.numError() + e := ec.compile(expr, false) + if e == nil && nerr == a.numError() { + log.Panicf("expression compilation failed without reporting errors") + } + return e +} + +// extractEffect separates out any effects that the expression may +// have, returning a function that will perform those effects and a +// new exprCompiler that is guaranteed to be side-effect free. These +// are the moral equivalents of "temp := expr" and "temp" (or "temp := +// &expr" and "*temp" for addressable exprs). Because this creates a +// temporary variable, the caller should create a temporary block for +// the compilation of this expression and the evaluation of the +// results. +func (a *expr) extractEffect(b *block, errOp string) (func(*Thread), *expr) { + // Create "&a" if a is addressable + rhs := a + if a.evalAddr != nil { + rhs = a.compileUnaryExpr(token.AND, rhs) + } + + // Create temp + ac, ok := a.checkAssign(a.pos, []*expr{rhs}, errOp, "") + if !ok { + return nil, nil + } + if len(ac.rmt.Elems) != 1 { + a.diag("multi-valued expression not allowed in %s", errOp) + return nil, nil + } + tempType := ac.rmt.Elems[0] + if tempType.isIdeal() { + // It's too bad we have to duplicate this rule. + switch { + case tempType.isInteger(): + tempType = IntType + case tempType.isFloat(): + tempType = Float64Type + default: + log.Panicf("unexpected ideal type %v", tempType) + } + } + temp := b.DefineTemp(tempType) + tempIdx := temp.Index + + // Create "temp := rhs" + assign := ac.compile(b, tempType) + if assign == nil { + log.Panicf("compileAssign type check failed") + } + + effect := func(t *Thread) { + tempVal := tempType.Zero() + t.f.Vars[tempIdx] = tempVal + assign(tempVal, t) + } + + // Generate "temp" or "*temp" + getTemp := a.compileVariable(0, temp) + if a.evalAddr == nil { + return effect, getTemp + } + + deref := a.compileStarExpr(getTemp) + if deref == nil { + return nil, nil + } + return effect, deref +} diff --git a/libgo/go/exp/eval/expr1.go b/libgo/go/exp/eval/expr1.go new file mode 100644 index 000000000..5d0e50000 --- /dev/null +++ b/libgo/go/exp/eval/expr1.go @@ -0,0 +1,1874 @@ +// This file is machine generated by gen.go. +// 6g gen.go && 6l gen.6 && ./6.out >expr1.go + +package eval + +import ( + "big" + "log" +) + +/* + * "As" functions. These retrieve evaluator functions from an + * expr, panicking if the requested evaluator has the wrong type. + */ +func (a *expr) asBool() func(*Thread) bool { + return a.eval.(func(*Thread) bool) +} +func (a *expr) asUint() func(*Thread) uint64 { + return a.eval.(func(*Thread) uint64) +} +func (a *expr) asInt() func(*Thread) int64 { + return a.eval.(func(*Thread) int64) +} +func (a *expr) asIdealInt() func() *big.Int { + return a.eval.(func() *big.Int) +} +func (a *expr) asFloat() func(*Thread) float64 { + return a.eval.(func(*Thread) float64) +} +func (a *expr) asIdealFloat() func() *big.Rat { + return a.eval.(func() *big.Rat) +} +func (a *expr) asString() func(*Thread) string { + return a.eval.(func(*Thread) string) +} +func (a *expr) asArray() func(*Thread) ArrayValue { + return a.eval.(func(*Thread) ArrayValue) +} +func (a *expr) asStruct() func(*Thread) StructValue { + return a.eval.(func(*Thread) StructValue) +} +func (a *expr) asPtr() func(*Thread) Value { + return a.eval.(func(*Thread) Value) +} +func (a *expr) asFunc() func(*Thread) Func { + return a.eval.(func(*Thread) Func) +} +func (a *expr) asSlice() func(*Thread) Slice { + return a.eval.(func(*Thread) Slice) +} +func (a *expr) asMap() func(*Thread) Map { + return a.eval.(func(*Thread) Map) +} +func (a *expr) asMulti() func(*Thread) []Value { + return a.eval.(func(*Thread) []Value) +} + +func (a *expr) asInterface() func(*Thread) interface{} { + switch sf := a.eval.(type) { + case func(t *Thread) bool: + return func(t *Thread) interface{} { return sf(t) } + case func(t *Thread) uint64: + return func(t *Thread) interface{} { return sf(t) } + case func(t *Thread) int64: + return func(t *Thread) interface{} { return sf(t) } + case func() *big.Int: + return func(*Thread) interface{} { return sf() } + case func(t *Thread) float64: + return func(t *Thread) interface{} { return sf(t) } + case func() *big.Rat: + return func(*Thread) interface{} { return sf() } + case func(t *Thread) string: + return func(t *Thread) interface{} { return sf(t) } + case func(t *Thread) ArrayValue: + return func(t *Thread) interface{} { return sf(t) } + case func(t *Thread) StructValue: + return func(t *Thread) interface{} { return sf(t) } + case func(t *Thread) Value: + return func(t *Thread) interface{} { return sf(t) } + case func(t *Thread) Func: + return func(t *Thread) interface{} { return sf(t) } + case func(t *Thread) Slice: + return func(t *Thread) interface{} { return sf(t) } + case func(t *Thread) Map: + return func(t *Thread) interface{} { return sf(t) } + default: + log.Panicf("unexpected expression node type %T at %v", a.eval, a.pos) + } + panic("fail") +} + +/* + * Operator generators. + */ + +func (a *expr) genConstant(v Value) { + switch a.t.lit().(type) { + case *boolType: + a.eval = func(t *Thread) bool { return v.(BoolValue).Get(t) } + case *uintType: + a.eval = func(t *Thread) uint64 { return v.(UintValue).Get(t) } + case *intType: + a.eval = func(t *Thread) int64 { return v.(IntValue).Get(t) } + case *idealIntType: + val := v.(IdealIntValue).Get() + a.eval = func() *big.Int { return val } + case *floatType: + a.eval = func(t *Thread) float64 { return v.(FloatValue).Get(t) } + case *idealFloatType: + val := v.(IdealFloatValue).Get() + a.eval = func() *big.Rat { return val } + case *stringType: + a.eval = func(t *Thread) string { return v.(StringValue).Get(t) } + case *ArrayType: + a.eval = func(t *Thread) ArrayValue { return v.(ArrayValue).Get(t) } + case *StructType: + a.eval = func(t *Thread) StructValue { return v.(StructValue).Get(t) } + case *PtrType: + a.eval = func(t *Thread) Value { return v.(PtrValue).Get(t) } + case *FuncType: + a.eval = func(t *Thread) Func { return v.(FuncValue).Get(t) } + case *SliceType: + a.eval = func(t *Thread) Slice { return v.(SliceValue).Get(t) } + case *MapType: + a.eval = func(t *Thread) Map { return v.(MapValue).Get(t) } + default: + log.Panicf("unexpected constant type %v at %v", a.t, a.pos) + } +} + +func (a *expr) genIdentOp(level, index int) { + a.evalAddr = func(t *Thread) Value { return t.f.Get(level, index) } + switch a.t.lit().(type) { + case *boolType: + a.eval = func(t *Thread) bool { return t.f.Get(level, index).(BoolValue).Get(t) } + case *uintType: + a.eval = func(t *Thread) uint64 { return t.f.Get(level, index).(UintValue).Get(t) } + case *intType: + a.eval = func(t *Thread) int64 { return t.f.Get(level, index).(IntValue).Get(t) } + case *floatType: + a.eval = func(t *Thread) float64 { return t.f.Get(level, index).(FloatValue).Get(t) } + case *stringType: + a.eval = func(t *Thread) string { return t.f.Get(level, index).(StringValue).Get(t) } + case *ArrayType: + a.eval = func(t *Thread) ArrayValue { return t.f.Get(level, index).(ArrayValue).Get(t) } + case *StructType: + a.eval = func(t *Thread) StructValue { return t.f.Get(level, index).(StructValue).Get(t) } + case *PtrType: + a.eval = func(t *Thread) Value { return t.f.Get(level, index).(PtrValue).Get(t) } + case *FuncType: + a.eval = func(t *Thread) Func { return t.f.Get(level, index).(FuncValue).Get(t) } + case *SliceType: + a.eval = func(t *Thread) Slice { return t.f.Get(level, index).(SliceValue).Get(t) } + case *MapType: + a.eval = func(t *Thread) Map { return t.f.Get(level, index).(MapValue).Get(t) } + default: + log.Panicf("unexpected identifier type %v at %v", a.t, a.pos) + } +} + +func (a *expr) genFuncCall(call func(t *Thread) []Value) { + a.exec = func(t *Thread) { call(t) } + switch a.t.lit().(type) { + case *boolType: + a.eval = func(t *Thread) bool { return call(t)[0].(BoolValue).Get(t) } + case *uintType: + a.eval = func(t *Thread) uint64 { return call(t)[0].(UintValue).Get(t) } + case *intType: + a.eval = func(t *Thread) int64 { return call(t)[0].(IntValue).Get(t) } + case *floatType: + a.eval = func(t *Thread) float64 { return call(t)[0].(FloatValue).Get(t) } + case *stringType: + a.eval = func(t *Thread) string { return call(t)[0].(StringValue).Get(t) } + case *ArrayType: + a.eval = func(t *Thread) ArrayValue { return call(t)[0].(ArrayValue).Get(t) } + case *StructType: + a.eval = func(t *Thread) StructValue { return call(t)[0].(StructValue).Get(t) } + case *PtrType: + a.eval = func(t *Thread) Value { return call(t)[0].(PtrValue).Get(t) } + case *FuncType: + a.eval = func(t *Thread) Func { return call(t)[0].(FuncValue).Get(t) } + case *SliceType: + a.eval = func(t *Thread) Slice { return call(t)[0].(SliceValue).Get(t) } + case *MapType: + a.eval = func(t *Thread) Map { return call(t)[0].(MapValue).Get(t) } + case *MultiType: + a.eval = func(t *Thread) []Value { return call(t) } + default: + log.Panicf("unexpected result type %v at %v", a.t, a.pos) + } +} + +func (a *expr) genValue(vf func(*Thread) Value) { + a.evalAddr = vf + switch a.t.lit().(type) { + case *boolType: + a.eval = func(t *Thread) bool { return vf(t).(BoolValue).Get(t) } + case *uintType: + a.eval = func(t *Thread) uint64 { return vf(t).(UintValue).Get(t) } + case *intType: + a.eval = func(t *Thread) int64 { return vf(t).(IntValue).Get(t) } + case *floatType: + a.eval = func(t *Thread) float64 { return vf(t).(FloatValue).Get(t) } + case *stringType: + a.eval = func(t *Thread) string { return vf(t).(StringValue).Get(t) } + case *ArrayType: + a.eval = func(t *Thread) ArrayValue { return vf(t).(ArrayValue).Get(t) } + case *StructType: + a.eval = func(t *Thread) StructValue { return vf(t).(StructValue).Get(t) } + case *PtrType: + a.eval = func(t *Thread) Value { return vf(t).(PtrValue).Get(t) } + case *FuncType: + a.eval = func(t *Thread) Func { return vf(t).(FuncValue).Get(t) } + case *SliceType: + a.eval = func(t *Thread) Slice { return vf(t).(SliceValue).Get(t) } + case *MapType: + a.eval = func(t *Thread) Map { return vf(t).(MapValue).Get(t) } + default: + log.Panicf("unexpected result type %v at %v", a.t, a.pos) + } +} + +func (a *expr) genUnaryOpNeg(v *expr) { + switch a.t.lit().(type) { + case *uintType: + vf := v.asUint() + a.eval = func(t *Thread) uint64 { v := vf(t); return -v } + case *intType: + vf := v.asInt() + a.eval = func(t *Thread) int64 { v := vf(t); return -v } + case *idealIntType: + val := v.asIdealInt()() + val.Neg(val) + a.eval = func() *big.Int { return val } + case *floatType: + vf := v.asFloat() + a.eval = func(t *Thread) float64 { v := vf(t); return -v } + case *idealFloatType: + val := v.asIdealFloat()() + val.Neg(val) + a.eval = func() *big.Rat { return val } + default: + log.Panicf("unexpected type %v at %v", a.t, a.pos) + } +} + +func (a *expr) genUnaryOpNot(v *expr) { + switch a.t.lit().(type) { + case *boolType: + vf := v.asBool() + a.eval = func(t *Thread) bool { v := vf(t); return !v } + default: + log.Panicf("unexpected type %v at %v", a.t, a.pos) + } +} + +func (a *expr) genUnaryOpXor(v *expr) { + switch a.t.lit().(type) { + case *uintType: + vf := v.asUint() + a.eval = func(t *Thread) uint64 { v := vf(t); return ^v } + case *intType: + vf := v.asInt() + a.eval = func(t *Thread) int64 { v := vf(t); return ^v } + case *idealIntType: + val := v.asIdealInt()() + val.Not(val) + a.eval = func() *big.Int { return val } + default: + log.Panicf("unexpected type %v at %v", a.t, a.pos) + } +} + +func (a *expr) genBinOpLogAnd(l, r *expr) { + lf := l.asBool() + rf := r.asBool() + a.eval = func(t *Thread) bool { return lf(t) && rf(t) } +} + +func (a *expr) genBinOpLogOr(l, r *expr) { + lf := l.asBool() + rf := r.asBool() + a.eval = func(t *Thread) bool { return lf(t) || rf(t) } +} + +func (a *expr) genBinOpAdd(l, r *expr) { + switch t := l.t.lit().(type) { + case *uintType: + lf := l.asUint() + rf := r.asUint() + switch t.Bits { + case 8: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + ret = l + r + return uint64(uint8(ret)) + } + case 16: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + ret = l + r + return uint64(uint16(ret)) + } + case 32: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + ret = l + r + return uint64(uint32(ret)) + } + case 64: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + ret = l + r + return uint64(uint64(ret)) + } + case 0: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + ret = l + r + return uint64(uint(ret)) + } + default: + log.Panicf("unexpected size %d in type %v at %v", t.Bits, t, a.pos) + } + case *intType: + lf := l.asInt() + rf := r.asInt() + switch t.Bits { + case 8: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + ret = l + r + return int64(int8(ret)) + } + case 16: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + ret = l + r + return int64(int16(ret)) + } + case 32: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + ret = l + r + return int64(int32(ret)) + } + case 64: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + ret = l + r + return int64(int64(ret)) + } + case 0: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + ret = l + r + return int64(int(ret)) + } + default: + log.Panicf("unexpected size %d in type %v at %v", t.Bits, t, a.pos) + } + case *idealIntType: + l := l.asIdealInt()() + r := r.asIdealInt()() + val := l.Add(l, r) + a.eval = func() *big.Int { return val } + case *floatType: + lf := l.asFloat() + rf := r.asFloat() + switch t.Bits { + case 32: + a.eval = func(t *Thread) float64 { + l, r := lf(t), rf(t) + var ret float64 + ret = l + r + return float64(float32(ret)) + } + case 64: + a.eval = func(t *Thread) float64 { + l, r := lf(t), rf(t) + var ret float64 + ret = l + r + return float64(float64(ret)) + } + default: + log.Panicf("unexpected size %d in type %v at %v", t.Bits, t, a.pos) + } + case *idealFloatType: + l := l.asIdealFloat()() + r := r.asIdealFloat()() + val := l.Add(l, r) + a.eval = func() *big.Rat { return val } + case *stringType: + lf := l.asString() + rf := r.asString() + a.eval = func(t *Thread) string { + l, r := lf(t), rf(t) + return l + r + } + default: + log.Panicf("unexpected type %v at %v", l.t, a.pos) + } +} + +func (a *expr) genBinOpSub(l, r *expr) { + switch t := l.t.lit().(type) { + case *uintType: + lf := l.asUint() + rf := r.asUint() + switch t.Bits { + case 8: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + ret = l - r + return uint64(uint8(ret)) + } + case 16: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + ret = l - r + return uint64(uint16(ret)) + } + case 32: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + ret = l - r + return uint64(uint32(ret)) + } + case 64: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + ret = l - r + return uint64(uint64(ret)) + } + case 0: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + ret = l - r + return uint64(uint(ret)) + } + default: + log.Panicf("unexpected size %d in type %v at %v", t.Bits, t, a.pos) + } + case *intType: + lf := l.asInt() + rf := r.asInt() + switch t.Bits { + case 8: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + ret = l - r + return int64(int8(ret)) + } + case 16: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + ret = l - r + return int64(int16(ret)) + } + case 32: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + ret = l - r + return int64(int32(ret)) + } + case 64: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + ret = l - r + return int64(int64(ret)) + } + case 0: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + ret = l - r + return int64(int(ret)) + } + default: + log.Panicf("unexpected size %d in type %v at %v", t.Bits, t, a.pos) + } + case *idealIntType: + l := l.asIdealInt()() + r := r.asIdealInt()() + val := l.Sub(l, r) + a.eval = func() *big.Int { return val } + case *floatType: + lf := l.asFloat() + rf := r.asFloat() + switch t.Bits { + case 32: + a.eval = func(t *Thread) float64 { + l, r := lf(t), rf(t) + var ret float64 + ret = l - r + return float64(float32(ret)) + } + case 64: + a.eval = func(t *Thread) float64 { + l, r := lf(t), rf(t) + var ret float64 + ret = l - r + return float64(float64(ret)) + } + default: + log.Panicf("unexpected size %d in type %v at %v", t.Bits, t, a.pos) + } + case *idealFloatType: + l := l.asIdealFloat()() + r := r.asIdealFloat()() + val := l.Sub(l, r) + a.eval = func() *big.Rat { return val } + default: + log.Panicf("unexpected type %v at %v", l.t, a.pos) + } +} + +func (a *expr) genBinOpMul(l, r *expr) { + switch t := l.t.lit().(type) { + case *uintType: + lf := l.asUint() + rf := r.asUint() + switch t.Bits { + case 8: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + ret = l * r + return uint64(uint8(ret)) + } + case 16: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + ret = l * r + return uint64(uint16(ret)) + } + case 32: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + ret = l * r + return uint64(uint32(ret)) + } + case 64: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + ret = l * r + return uint64(uint64(ret)) + } + case 0: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + ret = l * r + return uint64(uint(ret)) + } + default: + log.Panicf("unexpected size %d in type %v at %v", t.Bits, t, a.pos) + } + case *intType: + lf := l.asInt() + rf := r.asInt() + switch t.Bits { + case 8: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + ret = l * r + return int64(int8(ret)) + } + case 16: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + ret = l * r + return int64(int16(ret)) + } + case 32: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + ret = l * r + return int64(int32(ret)) + } + case 64: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + ret = l * r + return int64(int64(ret)) + } + case 0: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + ret = l * r + return int64(int(ret)) + } + default: + log.Panicf("unexpected size %d in type %v at %v", t.Bits, t, a.pos) + } + case *idealIntType: + l := l.asIdealInt()() + r := r.asIdealInt()() + val := l.Mul(l, r) + a.eval = func() *big.Int { return val } + case *floatType: + lf := l.asFloat() + rf := r.asFloat() + switch t.Bits { + case 32: + a.eval = func(t *Thread) float64 { + l, r := lf(t), rf(t) + var ret float64 + ret = l * r + return float64(float32(ret)) + } + case 64: + a.eval = func(t *Thread) float64 { + l, r := lf(t), rf(t) + var ret float64 + ret = l * r + return float64(float64(ret)) + } + default: + log.Panicf("unexpected size %d in type %v at %v", t.Bits, t, a.pos) + } + case *idealFloatType: + l := l.asIdealFloat()() + r := r.asIdealFloat()() + val := l.Mul(l, r) + a.eval = func() *big.Rat { return val } + default: + log.Panicf("unexpected type %v at %v", l.t, a.pos) + } +} + +func (a *expr) genBinOpQuo(l, r *expr) { + switch t := l.t.lit().(type) { + case *uintType: + lf := l.asUint() + rf := r.asUint() + switch t.Bits { + case 8: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + if r == 0 { + t.Abort(DivByZeroError{}) + } + ret = l / r + return uint64(uint8(ret)) + } + case 16: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + if r == 0 { + t.Abort(DivByZeroError{}) + } + ret = l / r + return uint64(uint16(ret)) + } + case 32: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + if r == 0 { + t.Abort(DivByZeroError{}) + } + ret = l / r + return uint64(uint32(ret)) + } + case 64: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + if r == 0 { + t.Abort(DivByZeroError{}) + } + ret = l / r + return uint64(uint64(ret)) + } + case 0: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + if r == 0 { + t.Abort(DivByZeroError{}) + } + ret = l / r + return uint64(uint(ret)) + } + default: + log.Panicf("unexpected size %d in type %v at %v", t.Bits, t, a.pos) + } + case *intType: + lf := l.asInt() + rf := r.asInt() + switch t.Bits { + case 8: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + if r == 0 { + t.Abort(DivByZeroError{}) + } + ret = l / r + return int64(int8(ret)) + } + case 16: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + if r == 0 { + t.Abort(DivByZeroError{}) + } + ret = l / r + return int64(int16(ret)) + } + case 32: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + if r == 0 { + t.Abort(DivByZeroError{}) + } + ret = l / r + return int64(int32(ret)) + } + case 64: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + if r == 0 { + t.Abort(DivByZeroError{}) + } + ret = l / r + return int64(int64(ret)) + } + case 0: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + if r == 0 { + t.Abort(DivByZeroError{}) + } + ret = l / r + return int64(int(ret)) + } + default: + log.Panicf("unexpected size %d in type %v at %v", t.Bits, t, a.pos) + } + case *idealIntType: + l := l.asIdealInt()() + r := r.asIdealInt()() + val := l.Quo(l, r) + a.eval = func() *big.Int { return val } + case *floatType: + lf := l.asFloat() + rf := r.asFloat() + switch t.Bits { + case 32: + a.eval = func(t *Thread) float64 { + l, r := lf(t), rf(t) + var ret float64 + if r == 0 { + t.Abort(DivByZeroError{}) + } + ret = l / r + return float64(float32(ret)) + } + case 64: + a.eval = func(t *Thread) float64 { + l, r := lf(t), rf(t) + var ret float64 + if r == 0 { + t.Abort(DivByZeroError{}) + } + ret = l / r + return float64(float64(ret)) + } + default: + log.Panicf("unexpected size %d in type %v at %v", t.Bits, t, a.pos) + } + case *idealFloatType: + l := l.asIdealFloat()() + r := r.asIdealFloat()() + val := l.Quo(l, r) + a.eval = func() *big.Rat { return val } + default: + log.Panicf("unexpected type %v at %v", l.t, a.pos) + } +} + +func (a *expr) genBinOpRem(l, r *expr) { + switch t := l.t.lit().(type) { + case *uintType: + lf := l.asUint() + rf := r.asUint() + switch t.Bits { + case 8: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + if r == 0 { + t.Abort(DivByZeroError{}) + } + ret = l % r + return uint64(uint8(ret)) + } + case 16: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + if r == 0 { + t.Abort(DivByZeroError{}) + } + ret = l % r + return uint64(uint16(ret)) + } + case 32: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + if r == 0 { + t.Abort(DivByZeroError{}) + } + ret = l % r + return uint64(uint32(ret)) + } + case 64: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + if r == 0 { + t.Abort(DivByZeroError{}) + } + ret = l % r + return uint64(uint64(ret)) + } + case 0: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + if r == 0 { + t.Abort(DivByZeroError{}) + } + ret = l % r + return uint64(uint(ret)) + } + default: + log.Panicf("unexpected size %d in type %v at %v", t.Bits, t, a.pos) + } + case *intType: + lf := l.asInt() + rf := r.asInt() + switch t.Bits { + case 8: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + if r == 0 { + t.Abort(DivByZeroError{}) + } + ret = l % r + return int64(int8(ret)) + } + case 16: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + if r == 0 { + t.Abort(DivByZeroError{}) + } + ret = l % r + return int64(int16(ret)) + } + case 32: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + if r == 0 { + t.Abort(DivByZeroError{}) + } + ret = l % r + return int64(int32(ret)) + } + case 64: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + if r == 0 { + t.Abort(DivByZeroError{}) + } + ret = l % r + return int64(int64(ret)) + } + case 0: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + if r == 0 { + t.Abort(DivByZeroError{}) + } + ret = l % r + return int64(int(ret)) + } + default: + log.Panicf("unexpected size %d in type %v at %v", t.Bits, t, a.pos) + } + case *idealIntType: + l := l.asIdealInt()() + r := r.asIdealInt()() + val := l.Rem(l, r) + a.eval = func() *big.Int { return val } + default: + log.Panicf("unexpected type %v at %v", l.t, a.pos) + } +} + +func (a *expr) genBinOpAnd(l, r *expr) { + switch t := l.t.lit().(type) { + case *uintType: + lf := l.asUint() + rf := r.asUint() + switch t.Bits { + case 8: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + ret = l & r + return uint64(uint8(ret)) + } + case 16: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + ret = l & r + return uint64(uint16(ret)) + } + case 32: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + ret = l & r + return uint64(uint32(ret)) + } + case 64: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + ret = l & r + return uint64(uint64(ret)) + } + case 0: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + ret = l & r + return uint64(uint(ret)) + } + default: + log.Panicf("unexpected size %d in type %v at %v", t.Bits, t, a.pos) + } + case *intType: + lf := l.asInt() + rf := r.asInt() + switch t.Bits { + case 8: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + ret = l & r + return int64(int8(ret)) + } + case 16: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + ret = l & r + return int64(int16(ret)) + } + case 32: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + ret = l & r + return int64(int32(ret)) + } + case 64: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + ret = l & r + return int64(int64(ret)) + } + case 0: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + ret = l & r + return int64(int(ret)) + } + default: + log.Panicf("unexpected size %d in type %v at %v", t.Bits, t, a.pos) + } + case *idealIntType: + l := l.asIdealInt()() + r := r.asIdealInt()() + val := l.And(l, r) + a.eval = func() *big.Int { return val } + default: + log.Panicf("unexpected type %v at %v", l.t, a.pos) + } +} + +func (a *expr) genBinOpOr(l, r *expr) { + switch t := l.t.lit().(type) { + case *uintType: + lf := l.asUint() + rf := r.asUint() + switch t.Bits { + case 8: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + ret = l | r + return uint64(uint8(ret)) + } + case 16: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + ret = l | r + return uint64(uint16(ret)) + } + case 32: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + ret = l | r + return uint64(uint32(ret)) + } + case 64: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + ret = l | r + return uint64(uint64(ret)) + } + case 0: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + ret = l | r + return uint64(uint(ret)) + } + default: + log.Panicf("unexpected size %d in type %v at %v", t.Bits, t, a.pos) + } + case *intType: + lf := l.asInt() + rf := r.asInt() + switch t.Bits { + case 8: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + ret = l | r + return int64(int8(ret)) + } + case 16: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + ret = l | r + return int64(int16(ret)) + } + case 32: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + ret = l | r + return int64(int32(ret)) + } + case 64: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + ret = l | r + return int64(int64(ret)) + } + case 0: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + ret = l | r + return int64(int(ret)) + } + default: + log.Panicf("unexpected size %d in type %v at %v", t.Bits, t, a.pos) + } + case *idealIntType: + l := l.asIdealInt()() + r := r.asIdealInt()() + val := l.Or(l, r) + a.eval = func() *big.Int { return val } + default: + log.Panicf("unexpected type %v at %v", l.t, a.pos) + } +} + +func (a *expr) genBinOpXor(l, r *expr) { + switch t := l.t.lit().(type) { + case *uintType: + lf := l.asUint() + rf := r.asUint() + switch t.Bits { + case 8: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + ret = l ^ r + return uint64(uint8(ret)) + } + case 16: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + ret = l ^ r + return uint64(uint16(ret)) + } + case 32: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + ret = l ^ r + return uint64(uint32(ret)) + } + case 64: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + ret = l ^ r + return uint64(uint64(ret)) + } + case 0: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + ret = l ^ r + return uint64(uint(ret)) + } + default: + log.Panicf("unexpected size %d in type %v at %v", t.Bits, t, a.pos) + } + case *intType: + lf := l.asInt() + rf := r.asInt() + switch t.Bits { + case 8: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + ret = l ^ r + return int64(int8(ret)) + } + case 16: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + ret = l ^ r + return int64(int16(ret)) + } + case 32: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + ret = l ^ r + return int64(int32(ret)) + } + case 64: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + ret = l ^ r + return int64(int64(ret)) + } + case 0: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + ret = l ^ r + return int64(int(ret)) + } + default: + log.Panicf("unexpected size %d in type %v at %v", t.Bits, t, a.pos) + } + case *idealIntType: + l := l.asIdealInt()() + r := r.asIdealInt()() + val := l.Xor(l, r) + a.eval = func() *big.Int { return val } + default: + log.Panicf("unexpected type %v at %v", l.t, a.pos) + } +} + +func (a *expr) genBinOpAndNot(l, r *expr) { + switch t := l.t.lit().(type) { + case *uintType: + lf := l.asUint() + rf := r.asUint() + switch t.Bits { + case 8: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + ret = l &^ r + return uint64(uint8(ret)) + } + case 16: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + ret = l &^ r + return uint64(uint16(ret)) + } + case 32: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + ret = l &^ r + return uint64(uint32(ret)) + } + case 64: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + ret = l &^ r + return uint64(uint64(ret)) + } + case 0: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + ret = l &^ r + return uint64(uint(ret)) + } + default: + log.Panicf("unexpected size %d in type %v at %v", t.Bits, t, a.pos) + } + case *intType: + lf := l.asInt() + rf := r.asInt() + switch t.Bits { + case 8: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + ret = l &^ r + return int64(int8(ret)) + } + case 16: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + ret = l &^ r + return int64(int16(ret)) + } + case 32: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + ret = l &^ r + return int64(int32(ret)) + } + case 64: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + ret = l &^ r + return int64(int64(ret)) + } + case 0: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + ret = l &^ r + return int64(int(ret)) + } + default: + log.Panicf("unexpected size %d in type %v at %v", t.Bits, t, a.pos) + } + case *idealIntType: + l := l.asIdealInt()() + r := r.asIdealInt()() + val := l.AndNot(l, r) + a.eval = func() *big.Int { return val } + default: + log.Panicf("unexpected type %v at %v", l.t, a.pos) + } +} + +func (a *expr) genBinOpShl(l, r *expr) { + switch t := l.t.lit().(type) { + case *uintType: + lf := l.asUint() + rf := r.asUint() + switch t.Bits { + case 8: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + ret = l << r + return uint64(uint8(ret)) + } + case 16: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + ret = l << r + return uint64(uint16(ret)) + } + case 32: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + ret = l << r + return uint64(uint32(ret)) + } + case 64: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + ret = l << r + return uint64(uint64(ret)) + } + case 0: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + ret = l << r + return uint64(uint(ret)) + } + default: + log.Panicf("unexpected size %d in type %v at %v", t.Bits, t, a.pos) + } + case *intType: + lf := l.asInt() + rf := r.asUint() + switch t.Bits { + case 8: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + ret = l << r + return int64(int8(ret)) + } + case 16: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + ret = l << r + return int64(int16(ret)) + } + case 32: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + ret = l << r + return int64(int32(ret)) + } + case 64: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + ret = l << r + return int64(int64(ret)) + } + case 0: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + ret = l << r + return int64(int(ret)) + } + default: + log.Panicf("unexpected size %d in type %v at %v", t.Bits, t, a.pos) + } + default: + log.Panicf("unexpected type %v at %v", l.t, a.pos) + } +} + +func (a *expr) genBinOpShr(l, r *expr) { + switch t := l.t.lit().(type) { + case *uintType: + lf := l.asUint() + rf := r.asUint() + switch t.Bits { + case 8: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + ret = l >> r + return uint64(uint8(ret)) + } + case 16: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + ret = l >> r + return uint64(uint16(ret)) + } + case 32: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + ret = l >> r + return uint64(uint32(ret)) + } + case 64: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + ret = l >> r + return uint64(uint64(ret)) + } + case 0: + a.eval = func(t *Thread) uint64 { + l, r := lf(t), rf(t) + var ret uint64 + ret = l >> r + return uint64(uint(ret)) + } + default: + log.Panicf("unexpected size %d in type %v at %v", t.Bits, t, a.pos) + } + case *intType: + lf := l.asInt() + rf := r.asUint() + switch t.Bits { + case 8: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + ret = l >> r + return int64(int8(ret)) + } + case 16: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + ret = l >> r + return int64(int16(ret)) + } + case 32: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + ret = l >> r + return int64(int32(ret)) + } + case 64: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + ret = l >> r + return int64(int64(ret)) + } + case 0: + a.eval = func(t *Thread) int64 { + l, r := lf(t), rf(t) + var ret int64 + ret = l >> r + return int64(int(ret)) + } + default: + log.Panicf("unexpected size %d in type %v at %v", t.Bits, t, a.pos) + } + default: + log.Panicf("unexpected type %v at %v", l.t, a.pos) + } +} + +func (a *expr) genBinOpLss(l, r *expr) { + switch t := l.t.lit().(type) { + case *uintType: + lf := l.asUint() + rf := r.asUint() + a.eval = func(t *Thread) bool { + l, r := lf(t), rf(t) + return l < r + } + case *intType: + lf := l.asInt() + rf := r.asInt() + a.eval = func(t *Thread) bool { + l, r := lf(t), rf(t) + return l < r + } + case *idealIntType: + l := l.asIdealInt()() + r := r.asIdealInt()() + val := l.Cmp(r) < 0 + a.eval = func(t *Thread) bool { return val } + case *floatType: + lf := l.asFloat() + rf := r.asFloat() + a.eval = func(t *Thread) bool { + l, r := lf(t), rf(t) + return l < r + } + case *idealFloatType: + l := l.asIdealFloat()() + r := r.asIdealFloat()() + val := l.Cmp(r) < 0 + a.eval = func(t *Thread) bool { return val } + case *stringType: + lf := l.asString() + rf := r.asString() + a.eval = func(t *Thread) bool { + l, r := lf(t), rf(t) + return l < r + } + default: + log.Panicf("unexpected type %v at %v", l.t, a.pos) + } +} + +func (a *expr) genBinOpGtr(l, r *expr) { + switch t := l.t.lit().(type) { + case *uintType: + lf := l.asUint() + rf := r.asUint() + a.eval = func(t *Thread) bool { + l, r := lf(t), rf(t) + return l > r + } + case *intType: + lf := l.asInt() + rf := r.asInt() + a.eval = func(t *Thread) bool { + l, r := lf(t), rf(t) + return l > r + } + case *idealIntType: + l := l.asIdealInt()() + r := r.asIdealInt()() + val := l.Cmp(r) > 0 + a.eval = func(t *Thread) bool { return val } + case *floatType: + lf := l.asFloat() + rf := r.asFloat() + a.eval = func(t *Thread) bool { + l, r := lf(t), rf(t) + return l > r + } + case *idealFloatType: + l := l.asIdealFloat()() + r := r.asIdealFloat()() + val := l.Cmp(r) > 0 + a.eval = func(t *Thread) bool { return val } + case *stringType: + lf := l.asString() + rf := r.asString() + a.eval = func(t *Thread) bool { + l, r := lf(t), rf(t) + return l > r + } + default: + log.Panicf("unexpected type %v at %v", l.t, a.pos) + } +} + +func (a *expr) genBinOpLeq(l, r *expr) { + switch t := l.t.lit().(type) { + case *uintType: + lf := l.asUint() + rf := r.asUint() + a.eval = func(t *Thread) bool { + l, r := lf(t), rf(t) + return l <= r + } + case *intType: + lf := l.asInt() + rf := r.asInt() + a.eval = func(t *Thread) bool { + l, r := lf(t), rf(t) + return l <= r + } + case *idealIntType: + l := l.asIdealInt()() + r := r.asIdealInt()() + val := l.Cmp(r) <= 0 + a.eval = func(t *Thread) bool { return val } + case *floatType: + lf := l.asFloat() + rf := r.asFloat() + a.eval = func(t *Thread) bool { + l, r := lf(t), rf(t) + return l <= r + } + case *idealFloatType: + l := l.asIdealFloat()() + r := r.asIdealFloat()() + val := l.Cmp(r) <= 0 + a.eval = func(t *Thread) bool { return val } + case *stringType: + lf := l.asString() + rf := r.asString() + a.eval = func(t *Thread) bool { + l, r := lf(t), rf(t) + return l <= r + } + default: + log.Panicf("unexpected type %v at %v", l.t, a.pos) + } +} + +func (a *expr) genBinOpGeq(l, r *expr) { + switch t := l.t.lit().(type) { + case *uintType: + lf := l.asUint() + rf := r.asUint() + a.eval = func(t *Thread) bool { + l, r := lf(t), rf(t) + return l >= r + } + case *intType: + lf := l.asInt() + rf := r.asInt() + a.eval = func(t *Thread) bool { + l, r := lf(t), rf(t) + return l >= r + } + case *idealIntType: + l := l.asIdealInt()() + r := r.asIdealInt()() + val := l.Cmp(r) >= 0 + a.eval = func(t *Thread) bool { return val } + case *floatType: + lf := l.asFloat() + rf := r.asFloat() + a.eval = func(t *Thread) bool { + l, r := lf(t), rf(t) + return l >= r + } + case *idealFloatType: + l := l.asIdealFloat()() + r := r.asIdealFloat()() + val := l.Cmp(r) >= 0 + a.eval = func(t *Thread) bool { return val } + case *stringType: + lf := l.asString() + rf := r.asString() + a.eval = func(t *Thread) bool { + l, r := lf(t), rf(t) + return l >= r + } + default: + log.Panicf("unexpected type %v at %v", l.t, a.pos) + } +} + +func (a *expr) genBinOpEql(l, r *expr) { + switch t := l.t.lit().(type) { + case *boolType: + lf := l.asBool() + rf := r.asBool() + a.eval = func(t *Thread) bool { + l, r := lf(t), rf(t) + return l == r + } + case *uintType: + lf := l.asUint() + rf := r.asUint() + a.eval = func(t *Thread) bool { + l, r := lf(t), rf(t) + return l == r + } + case *intType: + lf := l.asInt() + rf := r.asInt() + a.eval = func(t *Thread) bool { + l, r := lf(t), rf(t) + return l == r + } + case *idealIntType: + l := l.asIdealInt()() + r := r.asIdealInt()() + val := l.Cmp(r) == 0 + a.eval = func(t *Thread) bool { return val } + case *floatType: + lf := l.asFloat() + rf := r.asFloat() + a.eval = func(t *Thread) bool { + l, r := lf(t), rf(t) + return l == r + } + case *idealFloatType: + l := l.asIdealFloat()() + r := r.asIdealFloat()() + val := l.Cmp(r) == 0 + a.eval = func(t *Thread) bool { return val } + case *stringType: + lf := l.asString() + rf := r.asString() + a.eval = func(t *Thread) bool { + l, r := lf(t), rf(t) + return l == r + } + case *PtrType: + lf := l.asPtr() + rf := r.asPtr() + a.eval = func(t *Thread) bool { + l, r := lf(t), rf(t) + return l == r + } + case *FuncType: + lf := l.asFunc() + rf := r.asFunc() + a.eval = func(t *Thread) bool { + l, r := lf(t), rf(t) + return l == r + } + case *MapType: + lf := l.asMap() + rf := r.asMap() + a.eval = func(t *Thread) bool { + l, r := lf(t), rf(t) + return l == r + } + default: + log.Panicf("unexpected type %v at %v", l.t, a.pos) + } +} + +func (a *expr) genBinOpNeq(l, r *expr) { + switch t := l.t.lit().(type) { + case *boolType: + lf := l.asBool() + rf := r.asBool() + a.eval = func(t *Thread) bool { + l, r := lf(t), rf(t) + return l != r + } + case *uintType: + lf := l.asUint() + rf := r.asUint() + a.eval = func(t *Thread) bool { + l, r := lf(t), rf(t) + return l != r + } + case *intType: + lf := l.asInt() + rf := r.asInt() + a.eval = func(t *Thread) bool { + l, r := lf(t), rf(t) + return l != r + } + case *idealIntType: + l := l.asIdealInt()() + r := r.asIdealInt()() + val := l.Cmp(r) != 0 + a.eval = func(t *Thread) bool { return val } + case *floatType: + lf := l.asFloat() + rf := r.asFloat() + a.eval = func(t *Thread) bool { + l, r := lf(t), rf(t) + return l != r + } + case *idealFloatType: + l := l.asIdealFloat()() + r := r.asIdealFloat()() + val := l.Cmp(r) != 0 + a.eval = func(t *Thread) bool { return val } + case *stringType: + lf := l.asString() + rf := r.asString() + a.eval = func(t *Thread) bool { + l, r := lf(t), rf(t) + return l != r + } + case *PtrType: + lf := l.asPtr() + rf := r.asPtr() + a.eval = func(t *Thread) bool { + l, r := lf(t), rf(t) + return l != r + } + case *FuncType: + lf := l.asFunc() + rf := r.asFunc() + a.eval = func(t *Thread) bool { + l, r := lf(t), rf(t) + return l != r + } + case *MapType: + lf := l.asMap() + rf := r.asMap() + a.eval = func(t *Thread) bool { + l, r := lf(t), rf(t) + return l != r + } + default: + log.Panicf("unexpected type %v at %v", l.t, a.pos) + } +} + +func genAssign(lt Type, r *expr) func(lv Value, t *Thread) { + switch lt.lit().(type) { + case *boolType: + rf := r.asBool() + return func(lv Value, t *Thread) { lv.(BoolValue).Set(t, rf(t)) } + case *uintType: + rf := r.asUint() + return func(lv Value, t *Thread) { lv.(UintValue).Set(t, rf(t)) } + case *intType: + rf := r.asInt() + return func(lv Value, t *Thread) { lv.(IntValue).Set(t, rf(t)) } + case *floatType: + rf := r.asFloat() + return func(lv Value, t *Thread) { lv.(FloatValue).Set(t, rf(t)) } + case *stringType: + rf := r.asString() + return func(lv Value, t *Thread) { lv.(StringValue).Set(t, rf(t)) } + case *ArrayType: + rf := r.asArray() + return func(lv Value, t *Thread) { lv.Assign(t, rf(t)) } + case *StructType: + rf := r.asStruct() + return func(lv Value, t *Thread) { lv.Assign(t, rf(t)) } + case *PtrType: + rf := r.asPtr() + return func(lv Value, t *Thread) { lv.(PtrValue).Set(t, rf(t)) } + case *FuncType: + rf := r.asFunc() + return func(lv Value, t *Thread) { lv.(FuncValue).Set(t, rf(t)) } + case *SliceType: + rf := r.asSlice() + return func(lv Value, t *Thread) { lv.(SliceValue).Set(t, rf(t)) } + case *MapType: + rf := r.asMap() + return func(lv Value, t *Thread) { lv.(MapValue).Set(t, rf(t)) } + default: + log.Panicf("unexpected left operand type %v at %v", lt, r.pos) + } + panic("fail") +} diff --git a/libgo/go/exp/eval/expr_test.go b/libgo/go/exp/eval/expr_test.go new file mode 100644 index 000000000..0dbce4315 --- /dev/null +++ b/libgo/go/exp/eval/expr_test.go @@ -0,0 +1,355 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package eval + +import ( + "big" + "testing" +) + +var undefined = "undefined" +var typeAsExpr = "type .* used as expression" +var badCharLit = "character literal" +var unknownEscape = "unknown escape sequence" +var opTypes = "illegal (operand|argument) type|cannot index into" +var badAddrOf = "cannot take the address" +var constantTruncated = "constant [^ ]* truncated" +var constantUnderflows = "constant [^ ]* underflows" +var constantOverflows = "constant [^ ]* overflows" +var implLimit = "implementation limit" +var mustBeUnsigned = "must be unsigned" +var divByZero = "divide by zero" + +var hugeInteger = new(big.Int).Lsh(idealOne, 64) + +var exprTests = []test{ + Val("i", 1), + CErr("zzz", undefined), + // TODO(austin) Test variable in constant context + //CErr("t", typeAsExpr), + + Val("'a'", big.NewInt('a')), + Val("'\\uffff'", big.NewInt('\uffff')), + Val("'\\n'", big.NewInt('\n')), + CErr("''+x", badCharLit), + // Produces two parse errors + //CErr("'''", ""), + CErr("'\n'", badCharLit), + CErr("'\\z'", unknownEscape), + CErr("'ab'", badCharLit), + + Val("1.0", big.NewRat(1, 1)), + Val("1.", big.NewRat(1, 1)), + Val(".1", big.NewRat(1, 10)), + Val("1e2", big.NewRat(100, 1)), + + Val("\"abc\"", "abc"), + Val("\"\"", ""), + Val("\"\\n\\\"\"", "\n\""), + CErr("\"\\z\"", unknownEscape), + CErr("\"abc", "string not terminated"), + + Val("(i)", 1), + + Val("ai[0]", 1), + Val("(&ai)[0]", 1), + Val("ai[1]", 2), + Val("ai[i]", 2), + Val("ai[u]", 2), + CErr("ai[f]", opTypes), + CErr("ai[0][0]", opTypes), + CErr("ai[2]", "index 2 exceeds"), + CErr("ai[1+1]", "index 2 exceeds"), + CErr("ai[-1]", "negative index"), + RErr("ai[i+i]", "index 2 exceeds"), + RErr("ai[-i]", "negative index"), + CErr("i[0]", opTypes), + CErr("f[0]", opTypes), + + Val("aai[0][0]", 1), + Val("aai[1][1]", 4), + CErr("aai[2][0]", "index 2 exceeds"), + CErr("aai[0][2]", "index 2 exceeds"), + + Val("sli[0]", 1), + Val("sli[1]", 2), + CErr("sli[-1]", "negative index"), + RErr("sli[-i]", "negative index"), + RErr("sli[2]", "index 2 exceeds"), + + Val("s[0]", uint8('a')), + Val("s[1]", uint8('b')), + CErr("s[-1]", "negative index"), + RErr("s[-i]", "negative index"), + RErr("s[3]", "index 3 exceeds"), + + Val("ai[0:2]", vslice{varray{1, 2}, 2, 2}), + Val("ai[0:1]", vslice{varray{1, 2}, 1, 2}), + Val("ai[0:]", vslice{varray{1, 2}, 2, 2}), + Val("ai[i:]", vslice{varray{2}, 1, 1}), + + Val("sli[0:2]", vslice{varray{1, 2, 3}, 2, 3}), + Val("sli[0:i]", vslice{varray{1, 2, 3}, 1, 3}), + Val("sli[1:]", vslice{varray{2, 3}, 1, 2}), + + CErr("1(2)", "cannot call"), + CErr("fn(1,2)", "too many"), + CErr("fn()", "not enough"), + CErr("fn(true)", opTypes), + CErr("fn(true)", "function call"), + // Single argument functions don't say which argument. + //CErr("fn(true)", "argument 1"), + Val("fn(1)", 2), + Val("fn(1.0)", 2), + CErr("fn(1.5)", constantTruncated), + Val("fn(i)", 2), + CErr("fn(u)", opTypes), + + CErr("void()+2", opTypes), + CErr("oneTwo()+2", opTypes), + + Val("cap(ai)", 2), + Val("cap(&ai)", 2), + Val("cap(aai)", 2), + Val("cap(sli)", 3), + CErr("cap(0)", opTypes), + CErr("cap(i)", opTypes), + CErr("cap(s)", opTypes), + + Val("len(s)", 3), + Val("len(ai)", 2), + Val("len(&ai)", 2), + Val("len(ai[0:])", 2), + Val("len(ai[1:])", 1), + Val("len(ai[2:])", 0), + Val("len(aai)", 2), + Val("len(sli)", 2), + Val("len(sli[0:])", 2), + Val("len(sli[1:])", 1), + Val("len(sli[2:])", 0), + // TODO(austin) Test len of map + CErr("len(0)", opTypes), + CErr("len(i)", opTypes), + + CErr("*i", opTypes), + Val("*&i", 1), + Val("*&(i)", 1), + CErr("&1", badAddrOf), + CErr("&c", badAddrOf), + Val("*(&ai[0])", 1), + + Val("+1", big.NewInt(+1)), + Val("+1.0", big.NewRat(1, 1)), + Val("01.5", big.NewRat(15, 10)), + CErr("+\"x\"", opTypes), + + Val("-42", big.NewInt(-42)), + Val("-i", -1), + Val("-f", -1.0), + // 6g bug? + //Val("-(f-1)", -0.0), + CErr("-\"x\"", opTypes), + + // TODO(austin) Test unary ! + + Val("^2", big.NewInt(^2)), + Val("^(-2)", big.NewInt(^(-2))), + CErr("^2.0", opTypes), + CErr("^2.5", opTypes), + Val("^i", ^1), + Val("^u", ^uint(1)), + CErr("^f", opTypes), + + Val("1+i", 2), + Val("1+u", uint(2)), + Val("3.0+i", 4), + Val("1+1", big.NewInt(2)), + Val("f+f", 2.0), + Val("1+f", 2.0), + Val("1.0+1", big.NewRat(2, 1)), + Val("\"abc\" + \"def\"", "abcdef"), + CErr("i+u", opTypes), + CErr("-1+u", constantUnderflows), + // TODO(austin) Test named types + + Val("2-1", big.NewInt(1)), + Val("2.0-1", big.NewRat(1, 1)), + Val("f-2", -1.0), + Val("-0.0", big.NewRat(0, 1)), + Val("2*2", big.NewInt(4)), + Val("2*i", 2), + Val("3/2", big.NewInt(1)), + Val("3/i", 3), + CErr("1/0", divByZero), + CErr("1.0/0", divByZero), + RErr("i/0", divByZero), + Val("3%2", big.NewInt(1)), + Val("i%2", 1), + CErr("3%0", divByZero), + CErr("3.0%0", opTypes), + RErr("i%0", divByZero), + + // Examples from "Arithmetic operators" + Val("5/3", big.NewInt(1)), + Val("(i+4)/(i+2)", 1), + Val("5%3", big.NewInt(2)), + Val("(i+4)%(i+2)", 2), + Val("-5/3", big.NewInt(-1)), + Val("(i-6)/(i+2)", -1), + Val("-5%3", big.NewInt(-2)), + Val("(i-6)%(i+2)", -2), + Val("5/-3", big.NewInt(-1)), + Val("(i+4)/(i-4)", -1), + Val("5%-3", big.NewInt(2)), + Val("(i+4)%(i-4)", 2), + Val("-5/-3", big.NewInt(1)), + Val("(i-6)/(i-4)", 1), + Val("-5%-3", big.NewInt(-2)), + Val("(i-6)%(i-4)", -2), + + // Examples from "Arithmetic operators" + Val("11/4", big.NewInt(2)), + Val("(i+10)/4", 2), + Val("11%4", big.NewInt(3)), + Val("(i+10)%4", 3), + Val("11>>2", big.NewInt(2)), + Val("(i+10)>>2", 2), + Val("11&3", big.NewInt(3)), + Val("(i+10)&3", 3), + Val("-11/4", big.NewInt(-2)), + Val("(i-12)/4", -2), + Val("-11%4", big.NewInt(-3)), + Val("(i-12)%4", -3), + Val("-11>>2", big.NewInt(-3)), + Val("(i-12)>>2", -3), + Val("-11&3", big.NewInt(1)), + Val("(i-12)&3", 1), + + // TODO(austin) Test bit ops + + // For shift, we try nearly every combination of positive + // ideal int, negative ideal int, big ideal int, ideal + // fractional float, ideal non-fractional float, int, uint, + // and float. + Val("2<<2", big.NewInt(2<<2)), + CErr("2<<(-1)", constantUnderflows), + CErr("2<<0x10000000000000000", constantOverflows), + CErr("2<<2.5", constantTruncated), + Val("2<<2.0", big.NewInt(2<<2.0)), + CErr("2<, >= + Val("1<2", 1 < 2), + Val("1<=2", 1 <= 2), + Val("2<=2", 2 <= 2), + Val("1>2", 1 > 2), + Val("1>=2", 1 >= 2), + Val("2>=2", 2 >= 2), + + Val("i<2", 1 < 2), + Val("i<=2", 1 <= 2), + Val("i+1<=2", 2 <= 2), + Val("i>2", 1 > 2), + Val("i>=2", 1 >= 2), + Val("i+1>=2", 2 >= 2), + + Val("u<2", 1 < 2), + Val("f<2", 1 < 2), + + Val("s<\"b\"", true), + Val("s<\"a\"", false), + Val("s<=\"abc\"", true), + Val("s>\"aa\"", true), + Val("s>\"ac\"", false), + Val("s>=\"abc\"", true), + + CErr("i= b.scope.maxVars { + b.scope.maxVars = index + 1 + } + } + v := &Variable{token.NoPos, index, t, nil} + return v +} + +func (b *block) DefineConst(name string, pos token.Pos, t Type, v Value) (*Constant, Def) { + if prev, ok := b.defs[name]; ok { + return nil, prev + } + c := &Constant{pos, t, v} + b.defs[name] = c + return c, nil +} + +func (b *block) DefineType(name string, pos token.Pos, t Type) Type { + if _, ok := b.defs[name]; ok { + return nil + } + nt := &NamedType{pos, name, nil, true, make(map[string]Method)} + if t != nil { + nt.Complete(t) + } + b.defs[name] = nt + return nt +} + +func (b *block) Lookup(name string) (bl *block, level int, def Def) { + for b != nil { + if d, ok := b.defs[name]; ok { + return b, level, d + } + if b.outer != nil && b.scope != b.outer.scope { + level++ + } + b = b.outer + } + return nil, 0, nil +} + +func (s *Scope) NewFrame(outer *Frame) *Frame { return outer.child(s.maxVars) } + +/* + * Frames + */ + +type Frame struct { + Outer *Frame + Vars []Value +} + +func (f *Frame) Get(level int, index int) Value { + for ; level > 0; level-- { + f = f.Outer + } + return f.Vars[index] +} + +func (f *Frame) child(numVars int) *Frame { + // TODO(austin) This is probably rather expensive. All values + // require heap allocation and zeroing them when we execute a + // definition typically requires some computation. + return &Frame{f, make([]Value, numVars)} +} diff --git a/libgo/go/exp/eval/stmt.go b/libgo/go/exp/eval/stmt.go new file mode 100644 index 000000000..77ff066d0 --- /dev/null +++ b/libgo/go/exp/eval/stmt.go @@ -0,0 +1,1302 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package eval + +import ( + "big" + "log" + "go/ast" + "go/token" +) + +const ( + returnPC = ^uint(0) + badPC = ^uint(1) +) + +/* + * Statement compiler + */ + +type stmtCompiler struct { + *blockCompiler + pos token.Pos + // This statement's label, or nil if it is not labeled. + stmtLabel *label +} + +func (a *stmtCompiler) diag(format string, args ...interface{}) { + a.diagAt(a.pos, format, args...) +} + +/* + * Flow checker + */ + +type flowEnt struct { + // Whether this flow entry is conditional. If true, flow can + // continue to the next PC. + cond bool + // True if this will terminate flow (e.g., a return statement). + // cond must be false and jumps must be nil if this is true. + term bool + // PC's that can be reached from this flow entry. + jumps []*uint + // Whether this flow entry has been visited by reachesEnd. + visited bool +} + +type flowBlock struct { + // If this is a goto, the target label. + target string + // The inner-most block containing definitions. + block *block + // The numVars from each block leading to the root of the + // scope, starting at block. + numVars []int +} + +type flowBuf struct { + cb *codeBuf + // ents is a map from PC's to flow entries. Any PC missing + // from this map is assumed to reach only PC+1. + ents map[uint]*flowEnt + // gotos is a map from goto positions to information on the + // block at the point of the goto. + gotos map[token.Pos]*flowBlock + // labels is a map from label name to information on the block + // at the point of the label. labels are tracked by name, + // since mutliple labels at the same PC can have different + // blocks. + labels map[string]*flowBlock +} + +func newFlowBuf(cb *codeBuf) *flowBuf { + return &flowBuf{cb, make(map[uint]*flowEnt), make(map[token.Pos]*flowBlock), make(map[string]*flowBlock)} +} + +// put creates a flow control point for the next PC in the code buffer. +// This should be done before pushing the instruction into the code buffer. +func (f *flowBuf) put(cond bool, term bool, jumps []*uint) { + pc := f.cb.nextPC() + if ent, ok := f.ents[pc]; ok { + log.Panicf("Flow entry already exists at PC %d: %+v", pc, ent) + } + f.ents[pc] = &flowEnt{cond, term, jumps, false} +} + +// putTerm creates a flow control point at the next PC that +// unconditionally terminates execution. +func (f *flowBuf) putTerm() { f.put(false, true, nil) } + +// put1 creates a flow control point at the next PC that jumps to one +// PC and, if cond is true, can also continue to the PC following the +// next PC. +func (f *flowBuf) put1(cond bool, jumpPC *uint) { + f.put(cond, false, []*uint{jumpPC}) +} + +func newFlowBlock(target string, b *block) *flowBlock { + // Find the inner-most block containing definitions + for b.numVars == 0 && b.outer != nil && b.outer.scope == b.scope { + b = b.outer + } + + // Count parents leading to the root of the scope + n := 0 + for bp := b; bp.scope == b.scope; bp = bp.outer { + n++ + } + + // Capture numVars from each block to the root of the scope + numVars := make([]int, n) + i := 0 + for bp := b; i < n; bp = bp.outer { + numVars[i] = bp.numVars + i++ + } + + return &flowBlock{target, b, numVars} +} + +// putGoto captures the block at a goto statement. This should be +// called in addition to putting a flow control point. +func (f *flowBuf) putGoto(pos token.Pos, target string, b *block) { + f.gotos[pos] = newFlowBlock(target, b) +} + +// putLabel captures the block at a label. +func (f *flowBuf) putLabel(name string, b *block) { + f.labels[name] = newFlowBlock("", b) +} + +// reachesEnd returns true if the end of f's code buffer can be +// reached from the given program counter. Error reporting is the +// caller's responsibility. +func (f *flowBuf) reachesEnd(pc uint) bool { + endPC := f.cb.nextPC() + if pc > endPC { + log.Panicf("Reached bad PC %d past end PC %d", pc, endPC) + } + + for ; pc < endPC; pc++ { + ent, ok := f.ents[pc] + if !ok { + continue + } + + if ent.visited { + return false + } + ent.visited = true + + if ent.term { + return false + } + + // If anything can reach the end, we can reach the end + // from pc. + for _, j := range ent.jumps { + if f.reachesEnd(*j) { + return true + } + } + // If the jump was conditional, we can reach the next + // PC, so try reaching the end from it. + if ent.cond { + continue + } + return false + } + return true +} + +// gotosObeyScopes returns true if no goto statement causes any +// variables to come into scope that were not in scope at the point of +// the goto. Reports any errors using the given compiler. +func (f *flowBuf) gotosObeyScopes(a *compiler) { + for pos, src := range f.gotos { + tgt := f.labels[src.target] + + // The target block must be a parent of this block + numVars := src.numVars + b := src.block + for len(numVars) > 0 && b != tgt.block { + b = b.outer + numVars = numVars[1:] + } + if b != tgt.block { + // We jumped into a deeper block + a.diagAt(pos, "goto causes variables to come into scope") + return + } + + // There must be no variables in the target block that + // did not exist at the jump + tgtNumVars := tgt.numVars + for i := range numVars { + if tgtNumVars[i] > numVars[i] { + a.diagAt(pos, "goto causes variables to come into scope") + return + } + } + } +} + +/* + * Statement generation helpers + */ + +func (a *stmtCompiler) defineVar(ident *ast.Ident, t Type) *Variable { + v, prev := a.block.DefineVar(ident.Name, ident.Pos(), t) + if prev != nil { + if prev.Pos().IsValid() { + a.diagAt(ident.Pos(), "variable %s redeclared in this block\n\tprevious declaration at %s", ident.Name, a.fset.Position(prev.Pos())) + } else { + a.diagAt(ident.Pos(), "variable %s redeclared in this block", ident.Name) + } + return nil + } + + // Initialize the variable + index := v.Index + if v.Index >= 0 { + a.push(func(v *Thread) { v.f.Vars[index] = t.Zero() }) + } + return v +} + +// TODO(austin) Move doAssign to here + +/* + * Statement compiler + */ + +func (a *stmtCompiler) compile(s ast.Stmt) { + if a.block.inner != nil { + log.Panic("Child scope still entered") + } + + notimpl := false + switch s := s.(type) { + case *ast.BadStmt: + // Error already reported by parser. + a.silentErrors++ + + case *ast.DeclStmt: + a.compileDeclStmt(s) + + case *ast.EmptyStmt: + // Do nothing. + + case *ast.LabeledStmt: + a.compileLabeledStmt(s) + + case *ast.ExprStmt: + a.compileExprStmt(s) + + case *ast.IncDecStmt: + a.compileIncDecStmt(s) + + case *ast.AssignStmt: + a.compileAssignStmt(s) + + case *ast.GoStmt: + notimpl = true + + case *ast.DeferStmt: + notimpl = true + + case *ast.ReturnStmt: + a.compileReturnStmt(s) + + case *ast.BranchStmt: + a.compileBranchStmt(s) + + case *ast.BlockStmt: + a.compileBlockStmt(s) + + case *ast.IfStmt: + a.compileIfStmt(s) + + case *ast.CaseClause: + a.diag("case clause outside switch") + + case *ast.SwitchStmt: + a.compileSwitchStmt(s) + + case *ast.TypeCaseClause: + notimpl = true + + case *ast.TypeSwitchStmt: + notimpl = true + + case *ast.CommClause: + notimpl = true + + case *ast.SelectStmt: + notimpl = true + + case *ast.ForStmt: + a.compileForStmt(s) + + case *ast.RangeStmt: + notimpl = true + + default: + log.Panicf("unexpected ast node type %T", s) + } + + if notimpl { + a.diag("%T statment node not implemented", s) + } + + if a.block.inner != nil { + log.Panic("Forgot to exit child scope") + } +} + +func (a *stmtCompiler) compileDeclStmt(s *ast.DeclStmt) { + switch decl := s.Decl.(type) { + case *ast.BadDecl: + // Do nothing. Already reported by parser. + a.silentErrors++ + + case *ast.FuncDecl: + if !a.block.global { + log.Panic("FuncDecl at statement level") + } + + case *ast.GenDecl: + if decl.Tok == token.IMPORT && !a.block.global { + log.Panic("import at statement level") + } + + default: + log.Panicf("Unexpected Decl type %T", s.Decl) + } + a.compileDecl(s.Decl) +} + +func (a *stmtCompiler) compileVarDecl(decl *ast.GenDecl) { + for _, spec := range decl.Specs { + spec := spec.(*ast.ValueSpec) + if spec.Values == nil { + // Declaration without assignment + if spec.Type == nil { + // Parser should have caught + log.Panic("Type and Values nil") + } + t := a.compileType(a.block, spec.Type) + // Define placeholders even if type compile failed + for _, n := range spec.Names { + a.defineVar(n, t) + } + } else { + // Declaration with assignment + lhs := make([]ast.Expr, len(spec.Names)) + for i, n := range spec.Names { + lhs[i] = n + } + a.doAssign(lhs, spec.Values, decl.Tok, spec.Type) + } + } +} + +func (a *stmtCompiler) compileDecl(decl ast.Decl) { + switch d := decl.(type) { + case *ast.BadDecl: + // Do nothing. Already reported by parser. + a.silentErrors++ + + case *ast.FuncDecl: + decl := a.compileFuncType(a.block, d.Type) + if decl == nil { + return + } + // Declare and initialize v before compiling func + // so that body can refer to itself. + c, prev := a.block.DefineConst(d.Name.Name, a.pos, decl.Type, decl.Type.Zero()) + if prev != nil { + pos := prev.Pos() + if pos.IsValid() { + a.diagAt(d.Name.Pos(), "identifier %s redeclared in this block\n\tprevious declaration at %s", d.Name.Name, a.fset.Position(pos)) + } else { + a.diagAt(d.Name.Pos(), "identifier %s redeclared in this block", d.Name.Name) + } + } + fn := a.compileFunc(a.block, decl, d.Body) + if c == nil || fn == nil { + return + } + var zeroThread Thread + c.Value.(FuncValue).Set(nil, fn(&zeroThread)) + + case *ast.GenDecl: + switch d.Tok { + case token.IMPORT: + log.Panicf("%v not implemented", d.Tok) + case token.CONST: + log.Panicf("%v not implemented", d.Tok) + case token.TYPE: + a.compileTypeDecl(a.block, d) + case token.VAR: + a.compileVarDecl(d) + } + + default: + log.Panicf("Unexpected Decl type %T", decl) + } +} + +func (a *stmtCompiler) compileLabeledStmt(s *ast.LabeledStmt) { + // Define label + l, ok := a.labels[s.Label.Name] + if ok { + if l.resolved.IsValid() { + a.diag("label %s redeclared in this block\n\tprevious declaration at %s", s.Label.Name, a.fset.Position(l.resolved)) + } + } else { + pc := badPC + l = &label{name: s.Label.Name, gotoPC: &pc} + a.labels[l.name] = l + } + l.desc = "regular label" + l.resolved = s.Pos() + + // Set goto PC + *l.gotoPC = a.nextPC() + + // Define flow entry so we can check for jumps over declarations. + a.flow.putLabel(l.name, a.block) + + // Compile the statement. Reuse our stmtCompiler for simplicity. + sc := &stmtCompiler{a.blockCompiler, s.Stmt.Pos(), l} + sc.compile(s.Stmt) +} + +func (a *stmtCompiler) compileExprStmt(s *ast.ExprStmt) { + bc := a.enterChild() + defer bc.exit() + + e := a.compileExpr(bc.block, false, s.X) + if e == nil { + return + } + + if e.exec == nil { + a.diag("%s cannot be used as expression statement", e.desc) + return + } + + a.push(e.exec) +} + +func (a *stmtCompiler) compileIncDecStmt(s *ast.IncDecStmt) { + // Create temporary block for extractEffect + bc := a.enterChild() + defer bc.exit() + + l := a.compileExpr(bc.block, false, s.X) + if l == nil { + return + } + + if l.evalAddr == nil { + l.diag("cannot assign to %s", l.desc) + return + } + if !(l.t.isInteger() || l.t.isFloat()) { + l.diagOpType(s.Tok, l.t) + return + } + + var op token.Token + var desc string + switch s.Tok { + case token.INC: + op = token.ADD + desc = "increment statement" + case token.DEC: + op = token.SUB + desc = "decrement statement" + default: + log.Panicf("Unexpected IncDec token %v", s.Tok) + } + + effect, l := l.extractEffect(bc.block, desc) + + one := l.newExpr(IdealIntType, "constant") + one.pos = s.Pos() + one.eval = func() *big.Int { return big.NewInt(1) } + + binop := l.compileBinaryExpr(op, l, one) + if binop == nil { + return + } + + assign := a.compileAssign(s.Pos(), bc.block, l.t, []*expr{binop}, "", "") + if assign == nil { + log.Panicf("compileAssign type check failed") + } + + lf := l.evalAddr + a.push(func(v *Thread) { + effect(v) + assign(lf(v), v) + }) +} + +func (a *stmtCompiler) doAssign(lhs []ast.Expr, rhs []ast.Expr, tok token.Token, declTypeExpr ast.Expr) { + nerr := a.numError() + + // Compile right side first so we have the types when + // compiling the left side and so we don't see definitions + // made on the left side. + rs := make([]*expr, len(rhs)) + for i, re := range rhs { + rs[i] = a.compileExpr(a.block, false, re) + } + + errOp := "assignment" + if tok == token.DEFINE || tok == token.VAR { + errOp = "declaration" + } + ac, ok := a.checkAssign(a.pos, rs, errOp, "value") + ac.allowMapForms(len(lhs)) + + // If this is a definition and the LHS is too big, we won't be + // able to produce the usual error message because we can't + // begin to infer the types of the LHS. + if (tok == token.DEFINE || tok == token.VAR) && len(lhs) > len(ac.rmt.Elems) { + a.diag("not enough values for definition") + } + + // Compile left type if there is one + var declType Type + if declTypeExpr != nil { + declType = a.compileType(a.block, declTypeExpr) + } + + // Compile left side + ls := make([]*expr, len(lhs)) + nDefs := 0 + for i, le := range lhs { + // If this is a definition, get the identifier and its type + var ident *ast.Ident + var lt Type + switch tok { + case token.DEFINE: + // Check that it's an identifier + ident, ok = le.(*ast.Ident) + if !ok { + a.diagAt(le.Pos(), "left side of := must be a name") + // Suppress new defitions errors + nDefs++ + continue + } + + // Is this simply an assignment? + if _, ok := a.block.defs[ident.Name]; ok { + ident = nil + break + } + nDefs++ + + case token.VAR: + ident = le.(*ast.Ident) + } + + // If it's a definition, get or infer its type. + if ident != nil { + // Compute the identifier's type from the RHS + // type. We use the computed MultiType so we + // don't have to worry about unpacking. + switch { + case declTypeExpr != nil: + // We have a declaration type, use it. + // If declType is nil, we gave an + // error when we compiled it. + lt = declType + + case i >= len(ac.rmt.Elems): + // Define a placeholder. We already + // gave the "not enough" error above. + lt = nil + + case ac.rmt.Elems[i] == nil: + // We gave the error when we compiled + // the RHS. + lt = nil + + case ac.rmt.Elems[i].isIdeal(): + // If the type is absent and the + // corresponding expression is a + // constant expression of ideal + // integer or ideal float type, the + // type of the declared variable is + // int or float respectively. + switch { + case ac.rmt.Elems[i].isInteger(): + lt = IntType + case ac.rmt.Elems[i].isFloat(): + lt = Float64Type + default: + log.Panicf("unexpected ideal type %v", rs[i].t) + } + + default: + lt = ac.rmt.Elems[i] + } + } + + // If it's a definition, define the identifier + if ident != nil { + if a.defineVar(ident, lt) == nil { + continue + } + } + + // Compile LHS + ls[i] = a.compileExpr(a.block, false, le) + if ls[i] == nil { + continue + } + + if ls[i].evalMapValue != nil { + // Map indexes are not generally addressable, + // but they are assignable. + // + // TODO(austin) Now that the expression + // compiler uses semantic values, this might + // be easier to implement as a function call. + sub := ls[i] + ls[i] = ls[i].newExpr(sub.t, sub.desc) + ls[i].evalMapValue = sub.evalMapValue + mvf := sub.evalMapValue + et := sub.t + ls[i].evalAddr = func(t *Thread) Value { + m, k := mvf(t) + e := m.Elem(t, k) + if e == nil { + e = et.Zero() + m.SetElem(t, k, e) + } + return e + } + } else if ls[i].evalAddr == nil { + ls[i].diag("cannot assign to %s", ls[i].desc) + continue + } + } + + // A short variable declaration may redeclare variables + // provided they were originally declared in the same block + // with the same type, and at least one of the variables is + // new. + if tok == token.DEFINE && nDefs == 0 { + a.diag("at least one new variable must be declared") + return + } + + // If there have been errors, our arrays are full of nil's so + // get out of here now. + if nerr != a.numError() { + return + } + + // Check for 'a[x] = r, ok' + if len(ls) == 1 && len(rs) == 2 && ls[0].evalMapValue != nil { + a.diag("a[x] = r, ok form not implemented") + return + } + + // Create assigner + var lt Type + n := len(lhs) + if n == 1 { + lt = ls[0].t + } else { + lts := make([]Type, len(ls)) + for i, l := range ls { + if l != nil { + lts[i] = l.t + } + } + lt = NewMultiType(lts) + } + bc := a.enterChild() + defer bc.exit() + assign := ac.compile(bc.block, lt) + if assign == nil { + return + } + + // Compile + if n == 1 { + // Don't need temporaries and can avoid []Value. + lf := ls[0].evalAddr + a.push(func(t *Thread) { assign(lf(t), t) }) + } else if tok == token.VAR || (tok == token.DEFINE && nDefs == n) { + // Don't need temporaries + lfs := make([]func(*Thread) Value, n) + for i, l := range ls { + lfs[i] = l.evalAddr + } + a.push(func(t *Thread) { + dest := make([]Value, n) + for i, lf := range lfs { + dest[i] = lf(t) + } + assign(multiV(dest), t) + }) + } else { + // Need temporaries + lmt := lt.(*MultiType) + lfs := make([]func(*Thread) Value, n) + for i, l := range ls { + lfs[i] = l.evalAddr + } + a.push(func(t *Thread) { + temp := lmt.Zero().(multiV) + assign(temp, t) + // Copy to destination + for i := 0; i < n; i++ { + // TODO(austin) Need to evaluate LHS + // before RHS + lfs[i](t).Assign(t, temp[i]) + } + }) + } +} + +var assignOpToOp = map[token.Token]token.Token{ + token.ADD_ASSIGN: token.ADD, + token.SUB_ASSIGN: token.SUB, + token.MUL_ASSIGN: token.MUL, + token.QUO_ASSIGN: token.QUO, + token.REM_ASSIGN: token.REM, + + token.AND_ASSIGN: token.AND, + token.OR_ASSIGN: token.OR, + token.XOR_ASSIGN: token.XOR, + token.SHL_ASSIGN: token.SHL, + token.SHR_ASSIGN: token.SHR, + token.AND_NOT_ASSIGN: token.AND_NOT, +} + +func (a *stmtCompiler) doAssignOp(s *ast.AssignStmt) { + if len(s.Lhs) != 1 || len(s.Rhs) != 1 { + a.diag("tuple assignment cannot be combined with an arithmetic operation") + return + } + + // Create temporary block for extractEffect + bc := a.enterChild() + defer bc.exit() + + l := a.compileExpr(bc.block, false, s.Lhs[0]) + r := a.compileExpr(bc.block, false, s.Rhs[0]) + if l == nil || r == nil { + return + } + + if l.evalAddr == nil { + l.diag("cannot assign to %s", l.desc) + return + } + + effect, l := l.extractEffect(bc.block, "operator-assignment") + + binop := r.compileBinaryExpr(assignOpToOp[s.Tok], l, r) + if binop == nil { + return + } + + assign := a.compileAssign(s.Pos(), bc.block, l.t, []*expr{binop}, "assignment", "value") + if assign == nil { + log.Panicf("compileAssign type check failed") + } + + lf := l.evalAddr + a.push(func(t *Thread) { + effect(t) + assign(lf(t), t) + }) +} + +func (a *stmtCompiler) compileAssignStmt(s *ast.AssignStmt) { + switch s.Tok { + case token.ASSIGN, token.DEFINE: + a.doAssign(s.Lhs, s.Rhs, s.Tok, nil) + + default: + a.doAssignOp(s) + } +} + +func (a *stmtCompiler) compileReturnStmt(s *ast.ReturnStmt) { + if a.fnType == nil { + a.diag("cannot return at the top level") + return + } + + if len(s.Results) == 0 && (len(a.fnType.Out) == 0 || a.outVarsNamed) { + // Simple case. Simply exit from the function. + a.flow.putTerm() + a.push(func(v *Thread) { v.pc = returnPC }) + return + } + + bc := a.enterChild() + defer bc.exit() + + // Compile expressions + bad := false + rs := make([]*expr, len(s.Results)) + for i, re := range s.Results { + rs[i] = a.compileExpr(bc.block, false, re) + if rs[i] == nil { + bad = true + } + } + if bad { + return + } + + // Create assigner + + // However, if the expression list in the "return" statement + // is a single call to a multi-valued function, the values + // returned from the called function will be returned from + // this one. + assign := a.compileAssign(s.Pos(), bc.block, NewMultiType(a.fnType.Out), rs, "return", "value") + + // XXX(Spec) "The result types of the current function and the + // called function must match." Match is fuzzy. It should + // say that they must be assignment compatible. + + // Compile + start := len(a.fnType.In) + nout := len(a.fnType.Out) + a.flow.putTerm() + a.push(func(t *Thread) { + assign(multiV(t.f.Vars[start:start+nout]), t) + t.pc = returnPC + }) +} + +func (a *stmtCompiler) findLexicalLabel(name *ast.Ident, pred func(*label) bool, errOp, errCtx string) *label { + bc := a.blockCompiler + for ; bc != nil; bc = bc.parent { + if bc.label == nil { + continue + } + l := bc.label + if name == nil && pred(l) { + return l + } + if name != nil && l.name == name.Name { + if !pred(l) { + a.diag("cannot %s to %s %s", errOp, l.desc, l.name) + return nil + } + return l + } + } + if name == nil { + a.diag("%s outside %s", errOp, errCtx) + } else { + a.diag("%s label %s not defined", errOp, name.Name) + } + return nil +} + +func (a *stmtCompiler) compileBranchStmt(s *ast.BranchStmt) { + var pc *uint + + switch s.Tok { + case token.BREAK: + l := a.findLexicalLabel(s.Label, func(l *label) bool { return l.breakPC != nil }, "break", "for loop, switch, or select") + if l == nil { + return + } + pc = l.breakPC + + case token.CONTINUE: + l := a.findLexicalLabel(s.Label, func(l *label) bool { return l.continuePC != nil }, "continue", "for loop") + if l == nil { + return + } + pc = l.continuePC + + case token.GOTO: + l, ok := a.labels[s.Label.Name] + if !ok { + pc := badPC + l = &label{name: s.Label.Name, desc: "unresolved label", gotoPC: &pc, used: s.Pos()} + a.labels[l.name] = l + } + + pc = l.gotoPC + a.flow.putGoto(s.Pos(), l.name, a.block) + + case token.FALLTHROUGH: + a.diag("fallthrough outside switch") + return + + default: + log.Panic("Unexpected branch token %v", s.Tok) + } + + a.flow.put1(false, pc) + a.push(func(v *Thread) { v.pc = *pc }) +} + +func (a *stmtCompiler) compileBlockStmt(s *ast.BlockStmt) { + bc := a.enterChild() + bc.compileStmts(s) + bc.exit() +} + +func (a *stmtCompiler) compileIfStmt(s *ast.IfStmt) { + // The scope of any variables declared by [the init] statement + // extends to the end of the "if" statement and the variables + // are initialized once before the statement is entered. + // + // XXX(Spec) What this really wants to say is that there's an + // implicit scope wrapping every if, for, and switch + // statement. This is subtly different from what it actually + // says when there's a non-block else clause, because that + // else claus has to execute in a scope that is *not* the + // surrounding scope. + bc := a.enterChild() + defer bc.exit() + + // Compile init statement, if any + if s.Init != nil { + bc.compileStmt(s.Init) + } + + elsePC := badPC + endPC := badPC + + // Compile condition, if any. If there is no condition, we + // fall through to the body. + if s.Cond != nil { + e := bc.compileExpr(bc.block, false, s.Cond) + switch { + case e == nil: + // Error reported by compileExpr + case !e.t.isBoolean(): + e.diag("'if' condition must be boolean\n\t%v", e.t) + default: + eval := e.asBool() + a.flow.put1(true, &elsePC) + a.push(func(t *Thread) { + if !eval(t) { + t.pc = elsePC + } + }) + } + } + + // Compile body + body := bc.enterChild() + body.compileStmts(s.Body) + body.exit() + + // Compile else + if s.Else != nil { + // Skip over else if we executed the body + a.flow.put1(false, &endPC) + a.push(func(v *Thread) { v.pc = endPC }) + elsePC = a.nextPC() + bc.compileStmt(s.Else) + } else { + elsePC = a.nextPC() + } + endPC = a.nextPC() +} + +func (a *stmtCompiler) compileSwitchStmt(s *ast.SwitchStmt) { + // Create implicit scope around switch + bc := a.enterChild() + defer bc.exit() + + // Compile init statement, if any + if s.Init != nil { + bc.compileStmt(s.Init) + } + + // Compile condition, if any, and extract its effects + var cond *expr + condbc := bc.enterChild() + if s.Tag != nil { + e := condbc.compileExpr(condbc.block, false, s.Tag) + if e != nil { + var effect func(*Thread) + effect, cond = e.extractEffect(condbc.block, "switch") + a.push(effect) + } + } + + // Count cases + ncases := 0 + hasDefault := false + for _, c := range s.Body.List { + clause, ok := c.(*ast.CaseClause) + if !ok { + a.diagAt(clause.Pos(), "switch statement must contain case clauses") + continue + } + if clause.Values == nil { + if hasDefault { + a.diagAt(clause.Pos(), "switch statement contains more than one default case") + } + hasDefault = true + } else { + ncases += len(clause.Values) + } + } + + // Compile case expressions + cases := make([]func(*Thread) bool, ncases) + i := 0 + for _, c := range s.Body.List { + clause, ok := c.(*ast.CaseClause) + if !ok { + continue + } + for _, v := range clause.Values { + e := condbc.compileExpr(condbc.block, false, v) + switch { + case e == nil: + // Error reported by compileExpr + case cond == nil && !e.t.isBoolean(): + a.diagAt(v.Pos(), "'case' condition must be boolean") + case cond == nil: + cases[i] = e.asBool() + case cond != nil: + // Create comparison + // TOOD(austin) This produces bad error messages + compare := e.compileBinaryExpr(token.EQL, cond, e) + if compare != nil { + cases[i] = compare.asBool() + } + } + i++ + } + } + + // Emit condition + casePCs := make([]*uint, ncases+1) + endPC := badPC + + a.flow.put(false, false, casePCs) + a.push(func(t *Thread) { + for i, c := range cases { + if c(t) { + t.pc = *casePCs[i] + return + } + } + t.pc = *casePCs[ncases] + }) + condbc.exit() + + // Compile cases + i = 0 + for _, c := range s.Body.List { + clause, ok := c.(*ast.CaseClause) + if !ok { + continue + } + + // Save jump PC's + pc := a.nextPC() + if clause.Values != nil { + for _ = range clause.Values { + casePCs[i] = &pc + i++ + } + } else { + // Default clause + casePCs[ncases] = &pc + } + + // Compile body + fall := false + for j, s := range clause.Body { + if br, ok := s.(*ast.BranchStmt); ok && br.Tok == token.FALLTHROUGH { + // println("Found fallthrough"); + // It may be used only as the final + // non-empty statement in a case or + // default clause in an expression + // "switch" statement. + for _, s2 := range clause.Body[j+1:] { + // XXX(Spec) 6g also considers + // empty blocks to be empty + // statements. + if _, ok := s2.(*ast.EmptyStmt); !ok { + a.diagAt(s.Pos(), "fallthrough statement must be final statement in case") + break + } + } + fall = true + } else { + bc.compileStmt(s) + } + } + // Jump out of switch, unless there was a fallthrough + if !fall { + a.flow.put1(false, &endPC) + a.push(func(v *Thread) { v.pc = endPC }) + } + } + + // Get end PC + endPC = a.nextPC() + if !hasDefault { + casePCs[ncases] = &endPC + } +} + +func (a *stmtCompiler) compileForStmt(s *ast.ForStmt) { + // Wrap the entire for in a block. + bc := a.enterChild() + defer bc.exit() + + // Compile init statement, if any + if s.Init != nil { + bc.compileStmt(s.Init) + } + + bodyPC := badPC + postPC := badPC + checkPC := badPC + endPC := badPC + + // Jump to condition check. We generate slightly less code by + // placing the condition check after the body. + a.flow.put1(false, &checkPC) + a.push(func(v *Thread) { v.pc = checkPC }) + + // Compile body + bodyPC = a.nextPC() + body := bc.enterChild() + if a.stmtLabel != nil { + body.label = a.stmtLabel + } else { + body.label = &label{resolved: s.Pos()} + } + body.label.desc = "for loop" + body.label.breakPC = &endPC + body.label.continuePC = &postPC + body.compileStmts(s.Body) + body.exit() + + // Compile post, if any + postPC = a.nextPC() + if s.Post != nil { + // TODO(austin) Does the parser disallow short + // declarations in s.Post? + bc.compileStmt(s.Post) + } + + // Compile condition check, if any + checkPC = a.nextPC() + if s.Cond == nil { + // If the condition is absent, it is equivalent to true. + a.flow.put1(false, &bodyPC) + a.push(func(v *Thread) { v.pc = bodyPC }) + } else { + e := bc.compileExpr(bc.block, false, s.Cond) + switch { + case e == nil: + // Error reported by compileExpr + case !e.t.isBoolean(): + a.diag("'for' condition must be boolean\n\t%v", e.t) + default: + eval := e.asBool() + a.flow.put1(true, &bodyPC) + a.push(func(t *Thread) { + if eval(t) { + t.pc = bodyPC + } + }) + } + } + + endPC = a.nextPC() +} + +/* + * Block compiler + */ + +func (a *blockCompiler) compileStmt(s ast.Stmt) { + sc := &stmtCompiler{a, s.Pos(), nil} + sc.compile(s) +} + +func (a *blockCompiler) compileStmts(block *ast.BlockStmt) { + for _, sub := range block.List { + a.compileStmt(sub) + } +} + +func (a *blockCompiler) enterChild() *blockCompiler { + block := a.block.enterChild() + return &blockCompiler{ + funcCompiler: a.funcCompiler, + block: block, + parent: a, + } +} + +func (a *blockCompiler) exit() { a.block.exit() } + +/* + * Function compiler + */ + +func (a *compiler) compileFunc(b *block, decl *FuncDecl, body *ast.BlockStmt) func(*Thread) Func { + // Create body scope + // + // The scope of a parameter or result is the body of the + // corresponding function. + bodyScope := b.ChildScope() + defer bodyScope.exit() + for i, t := range decl.Type.In { + if decl.InNames[i] != nil { + bodyScope.DefineVar(decl.InNames[i].Name, decl.InNames[i].Pos(), t) + } else { + bodyScope.DefineTemp(t) + } + } + for i, t := range decl.Type.Out { + if decl.OutNames[i] != nil { + bodyScope.DefineVar(decl.OutNames[i].Name, decl.OutNames[i].Pos(), t) + } else { + bodyScope.DefineTemp(t) + } + } + + // Create block context + cb := newCodeBuf() + fc := &funcCompiler{ + compiler: a, + fnType: decl.Type, + outVarsNamed: len(decl.OutNames) > 0 && decl.OutNames[0] != nil, + codeBuf: cb, + flow: newFlowBuf(cb), + labels: make(map[string]*label), + } + bc := &blockCompiler{ + funcCompiler: fc, + block: bodyScope.block, + } + + // Compile body + nerr := a.numError() + bc.compileStmts(body) + fc.checkLabels() + if nerr != a.numError() { + return nil + } + + // Check that the body returned if necessary. We only check + // this if there were no errors compiling the body. + if len(decl.Type.Out) > 0 && fc.flow.reachesEnd(0) { + // XXX(Spec) Not specified. + a.diagAt(body.Rbrace, "function ends without a return statement") + return nil + } + + code := fc.get() + maxVars := bodyScope.maxVars + return func(t *Thread) Func { return &evalFunc{t.f, maxVars, code} } +} + +// Checks that labels were resolved and that all jumps obey scoping +// rules. Reports an error and set fc.err if any check fails. +func (a *funcCompiler) checkLabels() { + nerr := a.numError() + for _, l := range a.labels { + if !l.resolved.IsValid() { + a.diagAt(l.used, "label %s not defined", l.name) + } + } + if nerr != a.numError() { + // Don't check scopes if we have unresolved labels + return + } + + // Executing the "goto" statement must not cause any variables + // to come into scope that were not already in scope at the + // point of the goto. + a.flow.gotosObeyScopes(a.compiler) +} diff --git a/libgo/go/exp/eval/stmt_test.go b/libgo/go/exp/eval/stmt_test.go new file mode 100644 index 000000000..a14a288d9 --- /dev/null +++ b/libgo/go/exp/eval/stmt_test.go @@ -0,0 +1,343 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package eval + +import "testing" + +var atLeastOneDecl = "at least one new variable must be declared" + +var stmtTests = []test{ + // Short declarations + Val1("x := i", "x", 1), + Val1("x := f", "x", 1.0), + // Type defaulting + Val1("a := 42", "a", 42), + Val1("a := 1.0", "a", 1.0), + // Parallel assignment + Val2("a, b := 1, 2", "a", 1, "b", 2), + Val2("a, i := 1, 2", "a", 1, "i", 2), + CErr("a, i := 1, f", opTypes), + CErr("a, b := 1, 2, 3", "too many"), + CErr("a := 1, 2", "too many"), + CErr("a, b := 1", "not enough"), + // Mixed declarations + CErr("i := 1", atLeastOneDecl), + CErr("i, u := 1, 2", atLeastOneDecl), + Val2("i, x := 2, f", "i", 2, "x", 1.0), + // Various errors + CErr("1 := 2", "left side of := must be a name"), + CErr("c, a := 1, 1", "cannot assign"), + // Unpacking + Val2("x, y := oneTwo()", "x", 1, "y", 2), + CErr("x := oneTwo()", "too many"), + CErr("x, y, z := oneTwo()", "not enough"), + CErr("x, y := oneTwo(), 2", "multi-valued"), + CErr("x := oneTwo()+2", opTypes), + // TOOD(austin) This error message is weird + CErr("x := void()", "not enough"), + // Placeholders + CErr("x := 1+\"x\"; i=x+1", opTypes), + + // Assignment + Val1("i = 2", "i", 2), + Val1("(i) = 2", "i", 2), + CErr("1 = 2", "cannot assign"), + CErr("1-1 = 2", "- expression"), + Val1("i = 2.0", "i", 2), + CErr("i = 2.2", constantTruncated), + CErr("u = -2", constantUnderflows), + CErr("i = f", opTypes), + CErr("i, u = 0, f", opTypes), + CErr("i, u = 0, f", "value 2"), + Val2("i, i2 = i2, i", "i", 2, "i2", 1), + CErr("c = 1", "cannot assign"), + + Val1("x := &i; *x = 2", "i", 2), + + Val1("ai[0] = 42", "ai", varray{42, 2}), + Val1("aai[1] = ai; ai[0] = 42", "aai", varray{varray{1, 2}, varray{1, 2}}), + Val1("aai = aai2", "aai", varray{varray{5, 6}, varray{7, 8}}), + + // Assignment conversions + Run("var sl []int; sl = &ai"), + CErr("type ST []int; type AT *[2]int; var x AT = &ai; var y ST = x", opTypes), + Run("type ST []int; var y ST = &ai"), + Run("type AT *[2]int; var x AT = &ai; var y []int = x"), + + // Op-assignment + Val1("i += 2", "i", 3), + Val("i", 1), + Val1("f += 2", "f", 3.0), + CErr("2 += 2", "cannot assign"), + CErr("i, j += 2", "cannot be combined"), + CErr("i += 2, 3", "cannot be combined"), + Val2("s2 := s; s += \"def\"", "s2", "abc", "s", "abcdef"), + CErr("s += 1", opTypes), + // Single evaluation + Val2("ai[func()int{i+=1;return 0}()] *= 3; i2 = ai[0]", "i", 2, "i2", 3), + + // Type declarations + // Identifiers + Run("type T int"), + CErr("type T x", "undefined"), + CErr("type T c", "constant"), + CErr("type T i", "variable"), + CErr("type T T", "recursive"), + CErr("type T x; type U T; var v U; v = 1", "undefined"), + // Pointer types + Run("type T *int"), + Run("type T *T"), + // Array types + Run("type T [5]int"), + Run("type T [c+42/2]int"), + Run("type T [2.0]int"), + CErr("type T [i]int", "constant expression"), + CErr("type T [2.5]int", constantTruncated), + CErr("type T [-1]int", "negative"), + CErr("type T [2]T", "recursive"), + // Struct types + Run("type T struct { a int; b int }"), + Run("type T struct { a int; int }"), + Run("type T struct { x *T }"), + Run("type T int; type U struct { T }"), + CErr("type T *int; type U struct { T }", "embedded.*pointer"), + CErr("type T *struct { T }", "embedded.*pointer"), + CErr("type T struct { a int; a int }", " a .*redeclared.*:1:17"), + CErr("type T struct { int; int }", "int .*redeclared.*:1:17"), + CErr("type T struct { int int; int }", "int .*redeclared.*:1:17"), + Run("type T struct { x *struct { T } }"), + CErr("type T struct { x struct { T } }", "recursive"), + CErr("type T struct { x }; type U struct { T }", "undefined"), + // Function types + Run("type T func()"), + Run("type T func(a, b int) int"), + Run("type T func(a, b int) (x int, y int)"), + Run("type T func(a, a int) (a int, a int)"), + Run("type T func(a, b int) (x, y int)"), + Run("type T func(int, int) (int, int)"), + CErr("type T func(x); type U T", "undefined"), + CErr("type T func(a T)", "recursive"), + // Interface types + Run("type T interface {x(a, b int) int}"), + Run("type T interface {x(a, b int) int}; type U interface {T; y(c int)}"), + CErr("type T interface {x(a int); x()}", "method x redeclared"), + CErr("type T interface {x()}; type U interface {T; x()}", "method x redeclared"), + CErr("type T int; type U interface {T}", "embedded type"), + // Parens + Run("type T (int)"), + + // Variable declarations + Val2("var x int", "i", 1, "x", 0), + Val1("var x = 1", "x", 1), + Val1("var x = 1.0", "x", 1.0), + Val1("var x int = 1.0", "x", 1), + // Placeholders + CErr("var x foo; x = 1", "undefined"), + CErr("var x foo = 1; x = 1", "undefined"), + // Redeclaration + CErr("var i, x int", " i .*redeclared"), + CErr("var x int; var x int", " x .*redeclared.*:1:5"), + + // Expression statements + CErr("x := func(){ 1-1 }", "expression statement"), + CErr("x := func(){ 1-1 }", "- expression"), + Val1("fn(2)", "i", 1), + + // IncDec statements + Val1("i++", "i", 2), + Val1("i--", "i", 0), + Val1("u++", "u", uint(2)), + Val1("u--", "u", uint(0)), + Val1("f++", "f", 2.0), + Val1("f--", "f", 0.0), + // Single evaluation + Val2("ai[func()int{i+=1;return 0}()]++; i2 = ai[0]", "i", 2, "i2", 2), + // Operand types + CErr("s++", opTypes), + CErr("s++", "'\\+\\+'"), + CErr("2++", "cannot assign"), + CErr("c++", "cannot assign"), + + // Function scoping + Val1("fn1 := func() { i=2 }; fn1()", "i", 2), + Val1("fn1 := func() { i:=2 }; fn1()", "i", 1), + Val2("fn1 := func() int { i=2; i:=3; i=4; return i }; x := fn1()", "i", 2, "x", 4), + + // Basic returns + CErr("fn1 := func() int {}", "return"), + Run("fn1 := func() {}"), + CErr("fn1 := func() (r int) {}", "return"), + Val1("fn1 := func() (r int) {return}; i = fn1()", "i", 0), + Val1("fn1 := func() (r int) {r = 2; return}; i = fn1()", "i", 2), + Val1("fn1 := func() (r int) {return 2}; i = fn1()", "i", 2), + Val1("fn1 := func(int) int {return 2}; i = fn1(1)", "i", 2), + + // Multi-valued returns + Val2("fn1 := func() (bool, int) {return true, 2}; x, y := fn1()", "x", true, "y", 2), + CErr("fn1 := func() int {return}", "not enough values"), + CErr("fn1 := func() int {return 1,2}", "too many values"), + CErr("fn1 := func() {return 1}", "too many values"), + CErr("fn1 := func() (int,int,int) {return 1,2}", "not enough values"), + Val2("fn1 := func() (int, int) {return oneTwo()}; x, y := fn1()", "x", 1, "y", 2), + CErr("fn1 := func() int {return oneTwo()}", "too many values"), + CErr("fn1 := func() (int,int,int) {return oneTwo()}", "not enough values"), + Val1("fn1 := func(x,y int) int {return x+y}; x := fn1(oneTwo())", "x", 3), + + // Return control flow + Val2("fn1 := func(x *int) bool { *x = 2; return true; *x = 3; }; x := fn1(&i)", "i", 2, "x", true), + + // Break/continue/goto/fallthrough + CErr("break", "outside"), + CErr("break foo", "break.*foo.*not defined"), + CErr("continue", "outside"), + CErr("continue foo", "continue.*foo.*not defined"), + CErr("fallthrough", "outside"), + CErr("goto foo", "foo.*not defined"), + CErr(" foo: foo:;", "foo.*redeclared.*:1:2"), + Val1("i+=2; goto L; i+=4; L: i+=8", "i", 1+2+8), + // Return checking + CErr("fn1 := func() int { goto L; return 1; L: }", "return"), + Run("fn1 := func() int { L: goto L; i = 2 }"), + Run("fn1 := func() int { return 1; L: goto L }"), + // Scope checking + Run("fn1 := func() { { L: x:=1 }; goto L }"), + CErr("fn1 := func() { { x:=1; L: }; goto L }", "into scope"), + CErr("fn1 := func() { goto L; x:=1; L: }", "into scope"), + Run("fn1 := func() { goto L; { L: x:=1 } }"), + CErr("fn1 := func() { goto L; { x:=1; L: } }", "into scope"), + + // Blocks + CErr("fn1 := func() int {{}}", "return"), + Val1("fn1 := func() bool { { return true } }; b := fn1()", "b", true), + + // If + Val2("if true { i = 2 } else { i = 3 }; i2 = 4", "i", 2, "i2", 4), + Val2("if false { i = 2 } else { i = 3 }; i2 = 4", "i", 3, "i2", 4), + Val2("if i == i2 { i = 2 } else { i = 3 }; i2 = 4", "i", 3, "i2", 4), + // Omit optional parts + Val2("if { i = 2 } else { i = 3 }; i2 = 4", "i", 2, "i2", 4), + Val2("if true { i = 2 }; i2 = 4", "i", 2, "i2", 4), + Val2("if false { i = 2 }; i2 = 4", "i", 1, "i2", 4), + // Init + Val2("if x := true; x { i = 2 } else { i = 3 }; i2 = 4", "i", 2, "i2", 4), + Val2("if x := false; x { i = 2 } else { i = 3 }; i2 = 4", "i", 3, "i2", 4), + // Statement else + Val2("if true { i = 2 } else i = 3; i2 = 4", "i", 2, "i2", 4), + Val2("if false { i = 2 } else i = 3; i2 = 4", "i", 3, "i2", 4), + // Scoping + Val2("if true { i := 2 } else { i := 3 }; i2 = i", "i", 1, "i2", 1), + Val2("if false { i := 2 } else { i := 3 }; i2 = i", "i", 1, "i2", 1), + Val2("if false { i := 2 } else i := 3; i2 = i", "i", 1, "i2", 1), + CErr("if true { x := 2 }; x = 4", undefined), + Val2("if i := 2; true { i2 = i; i := 3 }", "i", 1, "i2", 2), + Val2("if i := 2; false {} else { i2 = i; i := 3 }", "i", 1, "i2", 2), + // Return checking + Run("fn1 := func() int { if true { return 1 } else { return 2 } }"), + Run("fn1 := func() int { if true { return 1 } else return 2 }"), + CErr("fn1 := func() int { if true { return 1 } else { } }", "return"), + CErr("fn1 := func() int { if true { } else { return 1 } }", "return"), + CErr("fn1 := func() int { if true { } else return 1 }", "return"), + CErr("fn1 := func() int { if true { } else { } }", "return"), + CErr("fn1 := func() int { if true { return 1 } }", "return"), + CErr("fn1 := func() int { if true { } }", "return"), + Run("fn1 := func() int { if true { }; return 1 }"), + CErr("fn1 := func() int { if { } }", "return"), + CErr("fn1 := func() int { if { } else { return 2 } }", "return"), + Run("fn1 := func() int { if { return 1 } }"), + Run("fn1 := func() int { if { return 1 } else { } }"), + Run("fn1 := func() int { if { return 1 } else { } }"), + + // Switch + Val1("switch { case false: i += 2; case true: i += 4; default: i += 8 }", "i", 1+4), + Val1("switch { default: i += 2; case false: i += 4; case true: i += 8 }", "i", 1+8), + CErr("switch { default: i += 2; default: i += 4 }", "more than one"), + Val1("switch false { case false: i += 2; case true: i += 4; default: i += 8 }", "i", 1+2), + CErr("switch s { case 1: }", opTypes), + CErr("switch ai { case ai: i += 2 }", opTypes), + Val1("switch 1.0 { case 1: i += 2; case 2: i += 4 }", "i", 1+2), + Val1("switch 1.5 { case 1: i += 2; case 2: i += 4 }", "i", 1), + CErr("switch oneTwo() {}", "multi-valued expression"), + Val1("switch 2 { case 1: i += 2; fallthrough; case 2: i += 4; fallthrough; case 3: i += 8; fallthrough }", "i", 1+4+8), + Val1("switch 5 { case 1: i += 2; fallthrough; default: i += 4; fallthrough; case 2: i += 8; fallthrough; case 3: i += 16; fallthrough }", "i", 1+4+8+16), + CErr("switch { case true: fallthrough; i += 2 }", "final statement"), + Val1("switch { case true: i += 2; fallthrough; ; ; case false: i += 4 }", "i", 1+2+4), + Val1("switch 2 { case 0, 1: i += 2; case 2, 3: i += 4 }", "i", 1+4), + Val2("switch func()int{i2++;return 5}() { case 1, 2: i += 2; case 4, 5: i += 4 }", "i", 1+4, "i2", 3), + Run("switch i { case i: }"), + // TODO(austin) Why doesn't this fail? + //CErr("case 1:", "XXX"), + + // For + Val2("for x := 1; x < 5; x++ { i+=x }; i2 = 4", "i", 11, "i2", 4), + Val2("for x := 1; x < 5; x++ { i+=x; break; i++ }; i2 = 4", "i", 2, "i2", 4), + Val2("for x := 1; x < 5; x++ { i+=x; continue; i++ }; i2 = 4", "i", 11, "i2", 4), + Val2("for i = 2; false; i = 3 { i = 4 }; i2 = 4", "i", 2, "i2", 4), + Val2("for i < 5 { i++ }; i2 = 4", "i", 5, "i2", 4), + Val2("for i < 0 { i++ }; i2 = 4", "i", 1, "i2", 4), + // Scoping + Val2("for i := 2; true; { i2 = i; i := 3; break }", "i", 1, "i2", 2), + // Labeled break/continue + Val1("L1: for { L2: for { i+=2; break L1; i+=4 }; i+=8 }", "i", 1+2), + Val1("L1: for { L2: for { i+=2; break L2; i+=4 }; i+=8; break; i+=16 }", "i", 1+2+8), + CErr("L1: { for { break L1 } }", "break.*not defined"), + CErr("L1: for {}; for { break L1 }", "break.*not defined"), + CErr("L1:; for { break L1 }", "break.*not defined"), + Val2("L1: for i = 0; i < 2; i++ { L2: for { i2++; continue L1; i2++ } }", "i", 2, "i2", 4), + CErr("L1: { for { continue L1 } }", "continue.*not defined"), + CErr("L1:; for { continue L1 }", "continue.*not defined"), + // Return checking + Run("fn1 := func() int{ for {} }"), + CErr("fn1 := func() int{ for true {} }", "return"), + CErr("fn1 := func() int{ for true {return 1} }", "return"), + CErr("fn1 := func() int{ for {break} }", "return"), + Run("fn1 := func() int{ for { for {break} } }"), + CErr("fn1 := func() int{ L1: for { for {break L1} } }", "return"), + Run("fn1 := func() int{ for true {}; return 1 }"), + + // Selectors + Val1("var x struct { a int; b int }; x.a = 42; i = x.a", "i", 42), + Val1("type T struct { x int }; var y struct { T }; y.x = 42; i = y.x", "i", 42), + Val2("type T struct { x int }; var y struct { T; x int }; y.x = 42; i = y.x; i2 = y.T.x", "i", 42, "i2", 0), + Run("type T struct { x int }; var y struct { *T }; a := func(){i=y.x}"), + CErr("type T struct { x int }; var x T; x.y = 42", "no field"), + CErr("type T struct { x int }; type U struct { x int }; var y struct { T; U }; y.x = 42", "ambiguous.*\tT\\.x\n\tU\\.x"), + CErr("type T struct { *T }; var x T; x.foo", "no field"), + + Val1("fib := func(int) int{return 0;}; fib = func(v int) int { if v < 2 { return 1 }; return fib(v-1)+fib(v-2) }; i = fib(20)", "i", 10946), + + // Make slice + Val2("x := make([]int, 2); x[0] = 42; i, i2 = x[0], x[1]", "i", 42, "i2", 0), + Val2("x := make([]int, 2); x[1] = 42; i, i2 = x[0], x[1]", "i", 0, "i2", 42), + RErr("x := make([]int, 2); x[-i] = 42", "negative index"), + RErr("x := make([]int, 2); x[2] = 42", "index 2 exceeds"), + Val2("x := make([]int, 2, 3); i, i2 = len(x), cap(x)", "i", 2, "i2", 3), + Val2("x := make([]int, 3, 2); i, i2 = len(x), cap(x)", "i", 3, "i2", 3), + RErr("x := make([]int, -i)", "negative length"), + RErr("x := make([]int, 2, -i)", "negative capacity"), + RErr("x := make([]int, 2, 3); x[2] = 42", "index 2 exceeds"), + CErr("x := make([]int, 2, 3, 4)", "too many"), + CErr("x := make([]int)", "not enough"), + + // TODO(austin) Test make map + + // Maps + Val1("x := make(map[int] int); x[1] = 42; i = x[1]", "i", 42), + Val2("x := make(map[int] int); x[1] = 42; i, y := x[1]", "i", 42, "y", true), + Val2("x := make(map[int] int); x[1] = 42; i, y := x[2]", "i", 0, "y", false), + // Not implemented + //Val1("x := make(map[int] int); x[1] = 42, true; i = x[1]", "i", 42), + //Val2("x := make(map[int] int); x[1] = 42; x[1] = 42, false; i, y := x[1]", "i", 0, "y", false), + Run("var x int; a := make(map[int] int); a[0], x = 1, 2"), + CErr("x := make(map[int] int); (func(a,b int){})(x[0])", "not enough"), + CErr("x := make(map[int] int); x[1] = oneTwo()", "too many"), + RErr("x := make(map[int] int); i = x[1]", "key '1' not found"), + + // Functions + Val2("func fib(n int) int { if n <= 2 { return n }; return fib(n-1) + fib(n-2) }", "fib(4)", 5, "fib(10)", 89), + Run("func f1(){}"), + Run2("func f1(){}", "f1()"), +} + +func TestStmt(t *testing.T) { runTests(t, "stmtTests", stmtTests) } diff --git a/libgo/go/exp/eval/type.go b/libgo/go/exp/eval/type.go new file mode 100644 index 000000000..3f272ce4b --- /dev/null +++ b/libgo/go/exp/eval/type.go @@ -0,0 +1,1252 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package eval + +import ( + "big" + "go/ast" + "go/token" + "log" + "reflect" + "sort" + "unsafe" // For Sizeof +) + + +// XXX(Spec) The type compatibility section is very confusing because +// it makes it seem like there are three distinct types of +// compatibility: plain compatibility, assignment compatibility, and +// comparison compatibility. As I understand it, there's really only +// assignment compatibility and comparison and conversion have some +// restrictions and have special meaning in some cases where the types +// are not otherwise assignment compatible. The comparison +// compatibility section is almost all about the semantics of +// comparison, not the type checking of it, so it would make much more +// sense in the comparison operators section. The compatibility and +// assignment compatibility sections should be rolled into one. + +type Type interface { + // compat returns whether this type is compatible with another + // type. If conv is false, this is normal compatibility, + // where two named types are compatible only if they are the + // same named type. If conv if true, this is conversion + // compatibility, where two named types are conversion + // compatible if their definitions are conversion compatible. + // + // TODO(austin) Deal with recursive types + compat(o Type, conv bool) bool + // lit returns this type's literal. If this is a named type, + // this is the unnamed underlying type. Otherwise, this is an + // identity operation. + lit() Type + // isBoolean returns true if this is a boolean type. + isBoolean() bool + // isInteger returns true if this is an integer type. + isInteger() bool + // isFloat returns true if this is a floating type. + isFloat() bool + // isIdeal returns true if this is an ideal int or float. + isIdeal() bool + // Zero returns a new zero value of this type. + Zero() Value + // String returns the string representation of this type. + String() string + // The position where this type was defined, if any. + Pos() token.Pos +} + +type BoundedType interface { + Type + // minVal returns the smallest value of this type. + minVal() *big.Rat + // maxVal returns the largest value of this type. + maxVal() *big.Rat +} + +var universePos = token.NoPos + +/* + * Type array maps. These are used to memoize composite types. + */ + +type typeArrayMapEntry struct { + key []Type + v interface{} + next *typeArrayMapEntry +} + +type typeArrayMap map[uintptr]*typeArrayMapEntry + +func hashTypeArray(key []Type) uintptr { + hash := uintptr(0) + for _, t := range key { + hash = hash * 33 + if t == nil { + continue + } + addr := reflect.NewValue(t).(*reflect.PtrValue).Get() + hash ^= addr + } + return hash +} + +func newTypeArrayMap() typeArrayMap { return make(map[uintptr]*typeArrayMapEntry) } + +func (m typeArrayMap) Get(key []Type) interface{} { + ent, ok := m[hashTypeArray(key)] + if !ok { + return nil + } + +nextEnt: + for ; ent != nil; ent = ent.next { + if len(key) != len(ent.key) { + continue + } + for i := 0; i < len(key); i++ { + if key[i] != ent.key[i] { + continue nextEnt + } + } + // Found it + return ent.v + } + + return nil +} + +func (m typeArrayMap) Put(key []Type, v interface{}) interface{} { + hash := hashTypeArray(key) + ent := m[hash] + + new := &typeArrayMapEntry{key, v, ent} + m[hash] = new + return v +} + +/* + * Common type + */ + +type commonType struct{} + +func (commonType) isBoolean() bool { return false } + +func (commonType) isInteger() bool { return false } + +func (commonType) isFloat() bool { return false } + +func (commonType) isIdeal() bool { return false } + +func (commonType) Pos() token.Pos { return token.NoPos } + +/* + * Bool + */ + +type boolType struct { + commonType +} + +var BoolType = universe.DefineType("bool", universePos, &boolType{}) + +func (t *boolType) compat(o Type, conv bool) bool { + _, ok := o.lit().(*boolType) + return ok +} + +func (t *boolType) lit() Type { return t } + +func (t *boolType) isBoolean() bool { return true } + +func (boolType) String() string { + // Use angle brackets as a convention for printing the + // underlying, unnamed type. This should only show up in + // debug output. + return "" +} + +func (t *boolType) Zero() Value { + res := boolV(false) + return &res +} + +/* + * Uint + */ + +type uintType struct { + commonType + + // 0 for architecture-dependent types + Bits uint + // true for uintptr, false for all others + Ptr bool + name string +} + +var ( + Uint8Type = universe.DefineType("uint8", universePos, &uintType{commonType{}, 8, false, "uint8"}) + Uint16Type = universe.DefineType("uint16", universePos, &uintType{commonType{}, 16, false, "uint16"}) + Uint32Type = universe.DefineType("uint32", universePos, &uintType{commonType{}, 32, false, "uint32"}) + Uint64Type = universe.DefineType("uint64", universePos, &uintType{commonType{}, 64, false, "uint64"}) + + UintType = universe.DefineType("uint", universePos, &uintType{commonType{}, 0, false, "uint"}) + UintptrType = universe.DefineType("uintptr", universePos, &uintType{commonType{}, 0, true, "uintptr"}) +) + +func (t *uintType) compat(o Type, conv bool) bool { + t2, ok := o.lit().(*uintType) + return ok && t == t2 +} + +func (t *uintType) lit() Type { return t } + +func (t *uintType) isInteger() bool { return true } + +func (t *uintType) String() string { return "<" + t.name + ">" } + +func (t *uintType) Zero() Value { + switch t.Bits { + case 0: + if t.Ptr { + res := uintptrV(0) + return &res + } else { + res := uintV(0) + return &res + } + case 8: + res := uint8V(0) + return &res + case 16: + res := uint16V(0) + return &res + case 32: + res := uint32V(0) + return &res + case 64: + res := uint64V(0) + return &res + } + panic("unexpected uint bit count") +} + +func (t *uintType) minVal() *big.Rat { return big.NewRat(0, 1) } + +func (t *uintType) maxVal() *big.Rat { + bits := t.Bits + if bits == 0 { + if t.Ptr { + bits = uint(8 * unsafe.Sizeof(uintptr(0))) + } else { + bits = uint(8 * unsafe.Sizeof(uint(0))) + } + } + numer := big.NewInt(1) + numer.Lsh(numer, bits) + numer.Sub(numer, idealOne) + return new(big.Rat).SetInt(numer) +} + +/* + * Int + */ + +type intType struct { + commonType + + // XXX(Spec) Numeric types: "There is also a set of + // architecture-independent basic numeric types whose size + // depends on the architecture." Should that be + // architecture-dependent? + + // 0 for architecture-dependent types + Bits uint + name string +} + +var ( + Int8Type = universe.DefineType("int8", universePos, &intType{commonType{}, 8, "int8"}) + Int16Type = universe.DefineType("int16", universePos, &intType{commonType{}, 16, "int16"}) + Int32Type = universe.DefineType("int32", universePos, &intType{commonType{}, 32, "int32"}) + Int64Type = universe.DefineType("int64", universePos, &intType{commonType{}, 64, "int64"}) + + IntType = universe.DefineType("int", universePos, &intType{commonType{}, 0, "int"}) +) + +func (t *intType) compat(o Type, conv bool) bool { + t2, ok := o.lit().(*intType) + return ok && t == t2 +} + +func (t *intType) lit() Type { return t } + +func (t *intType) isInteger() bool { return true } + +func (t *intType) String() string { return "<" + t.name + ">" } + +func (t *intType) Zero() Value { + switch t.Bits { + case 8: + res := int8V(0) + return &res + case 16: + res := int16V(0) + return &res + case 32: + res := int32V(0) + return &res + case 64: + res := int64V(0) + return &res + + case 0: + res := intV(0) + return &res + } + panic("unexpected int bit count") +} + +func (t *intType) minVal() *big.Rat { + bits := t.Bits + if bits == 0 { + bits = uint(8 * unsafe.Sizeof(int(0))) + } + numer := big.NewInt(-1) + numer.Lsh(numer, bits-1) + return new(big.Rat).SetInt(numer) +} + +func (t *intType) maxVal() *big.Rat { + bits := t.Bits + if bits == 0 { + bits = uint(8 * unsafe.Sizeof(int(0))) + } + numer := big.NewInt(1) + numer.Lsh(numer, bits-1) + numer.Sub(numer, idealOne) + return new(big.Rat).SetInt(numer) +} + +/* + * Ideal int + */ + +type idealIntType struct { + commonType +} + +var IdealIntType Type = &idealIntType{} + +func (t *idealIntType) compat(o Type, conv bool) bool { + _, ok := o.lit().(*idealIntType) + return ok +} + +func (t *idealIntType) lit() Type { return t } + +func (t *idealIntType) isInteger() bool { return true } + +func (t *idealIntType) isIdeal() bool { return true } + +func (t *idealIntType) String() string { return "ideal integer" } + +func (t *idealIntType) Zero() Value { return &idealIntV{idealZero} } + +/* + * Float + */ + +type floatType struct { + commonType + + // 0 for architecture-dependent type + Bits uint + + name string +} + +var ( + Float32Type = universe.DefineType("float32", universePos, &floatType{commonType{}, 32, "float32"}) + Float64Type = universe.DefineType("float64", universePos, &floatType{commonType{}, 64, "float64"}) +) + +func (t *floatType) compat(o Type, conv bool) bool { + t2, ok := o.lit().(*floatType) + return ok && t == t2 +} + +func (t *floatType) lit() Type { return t } + +func (t *floatType) isFloat() bool { return true } + +func (t *floatType) String() string { return "<" + t.name + ">" } + +func (t *floatType) Zero() Value { + switch t.Bits { + case 32: + res := float32V(0) + return &res + case 64: + res := float64V(0) + return &res + } + panic("unexpected float bit count") +} + +var maxFloat32Val *big.Rat +var maxFloat64Val *big.Rat +var minFloat32Val *big.Rat +var minFloat64Val *big.Rat + +func (t *floatType) minVal() *big.Rat { + bits := t.Bits + switch bits { + case 32: + return minFloat32Val + case 64: + return minFloat64Val + } + log.Panicf("unexpected floating point bit count: %d", bits) + panic("unreachable") +} + +func (t *floatType) maxVal() *big.Rat { + bits := t.Bits + switch bits { + case 32: + return maxFloat32Val + case 64: + return maxFloat64Val + } + log.Panicf("unexpected floating point bit count: %d", bits) + panic("unreachable") +} + +/* + * Ideal float + */ + +type idealFloatType struct { + commonType +} + +var IdealFloatType Type = &idealFloatType{} + +func (t *idealFloatType) compat(o Type, conv bool) bool { + _, ok := o.lit().(*idealFloatType) + return ok +} + +func (t *idealFloatType) lit() Type { return t } + +func (t *idealFloatType) isFloat() bool { return true } + +func (t *idealFloatType) isIdeal() bool { return true } + +func (t *idealFloatType) String() string { return "ideal float" } + +func (t *idealFloatType) Zero() Value { return &idealFloatV{big.NewRat(0, 1)} } + +/* + * String + */ + +type stringType struct { + commonType +} + +var StringType = universe.DefineType("string", universePos, &stringType{}) + +func (t *stringType) compat(o Type, conv bool) bool { + _, ok := o.lit().(*stringType) + return ok +} + +func (t *stringType) lit() Type { return t } + +func (t *stringType) String() string { return "" } + +func (t *stringType) Zero() Value { + res := stringV("") + return &res +} + +/* + * Array + */ + +type ArrayType struct { + commonType + Len int64 + Elem Type +} + +var arrayTypes = make(map[int64]map[Type]*ArrayType) + +// Two array types are identical if they have identical element types +// and the same array length. + +func NewArrayType(len int64, elem Type) *ArrayType { + ts, ok := arrayTypes[len] + if !ok { + ts = make(map[Type]*ArrayType) + arrayTypes[len] = ts + } + t, ok := ts[elem] + if !ok { + t = &ArrayType{commonType{}, len, elem} + ts[elem] = t + } + return t +} + +func (t *ArrayType) compat(o Type, conv bool) bool { + t2, ok := o.lit().(*ArrayType) + if !ok { + return false + } + return t.Len == t2.Len && t.Elem.compat(t2.Elem, conv) +} + +func (t *ArrayType) lit() Type { return t } + +func (t *ArrayType) String() string { return "[]" + t.Elem.String() } + +func (t *ArrayType) Zero() Value { + res := arrayV(make([]Value, t.Len)) + // TODO(austin) It's unfortunate that each element is + // separately heap allocated. We could add ZeroArray to + // everything, though that doesn't help with multidimensional + // arrays. Or we could do something unsafe. We'll have this + // same problem with structs. + for i := int64(0); i < t.Len; i++ { + res[i] = t.Elem.Zero() + } + return &res +} + +/* + * Struct + */ + +type StructField struct { + Name string + Type Type + Anonymous bool +} + +type StructType struct { + commonType + Elems []StructField +} + +var structTypes = newTypeArrayMap() + +// Two struct types are identical if they have the same sequence of +// fields, and if corresponding fields have the same names and +// identical types. Two anonymous fields are considered to have the +// same name. + +func NewStructType(fields []StructField) *StructType { + // Start by looking up just the types + fts := make([]Type, len(fields)) + for i, f := range fields { + fts[i] = f.Type + } + tMapI := structTypes.Get(fts) + if tMapI == nil { + tMapI = structTypes.Put(fts, make(map[string]*StructType)) + } + tMap := tMapI.(map[string]*StructType) + + // Construct key for field names + key := "" + for _, f := range fields { + // XXX(Spec) It's not clear if struct { T } and struct + // { T T } are either identical or compatible. The + // "Struct Types" section says that the name of that + // field is "T", which suggests that they are + // identical, but it really means that it's the name + // for the purpose of selector expressions and nothing + // else. We decided that they should be neither + // identical or compatible. + if f.Anonymous { + key += "!" + } + key += f.Name + " " + } + + // XXX(Spec) Do the tags also have to be identical for the + // types to be identical? I certainly hope so, because + // otherwise, this is the only case where two distinct type + // objects can represent identical types. + + t, ok := tMap[key] + if !ok { + // Create new struct type + t = &StructType{commonType{}, fields} + tMap[key] = t + } + return t +} + +func (t *StructType) compat(o Type, conv bool) bool { + t2, ok := o.lit().(*StructType) + if !ok { + return false + } + if len(t.Elems) != len(t2.Elems) { + return false + } + for i, e := range t.Elems { + e2 := t2.Elems[i] + // XXX(Spec) An anonymous and a non-anonymous field + // are neither identical nor compatible. + if e.Anonymous != e2.Anonymous || + (!e.Anonymous && e.Name != e2.Name) || + !e.Type.compat(e2.Type, conv) { + return false + } + } + return true +} + +func (t *StructType) lit() Type { return t } + +func (t *StructType) String() string { + s := "struct {" + for i, f := range t.Elems { + if i > 0 { + s += "; " + } + if !f.Anonymous { + s += f.Name + " " + } + s += f.Type.String() + } + return s + "}" +} + +func (t *StructType) Zero() Value { + res := structV(make([]Value, len(t.Elems))) + for i, f := range t.Elems { + res[i] = f.Type.Zero() + } + return &res +} + +/* + * Pointer + */ + +type PtrType struct { + commonType + Elem Type +} + +var ptrTypes = make(map[Type]*PtrType) + +// Two pointer types are identical if they have identical base types. + +func NewPtrType(elem Type) *PtrType { + t, ok := ptrTypes[elem] + if !ok { + t = &PtrType{commonType{}, elem} + ptrTypes[elem] = t + } + return t +} + +func (t *PtrType) compat(o Type, conv bool) bool { + t2, ok := o.lit().(*PtrType) + if !ok { + return false + } + return t.Elem.compat(t2.Elem, conv) +} + +func (t *PtrType) lit() Type { return t } + +func (t *PtrType) String() string { return "*" + t.Elem.String() } + +func (t *PtrType) Zero() Value { return &ptrV{nil} } + +/* + * Function + */ + +type FuncType struct { + commonType + // TODO(austin) Separate receiver Type for methods? + In []Type + Variadic bool + Out []Type + builtin string +} + +var funcTypes = newTypeArrayMap() +var variadicFuncTypes = newTypeArrayMap() + +// Create singleton function types for magic built-in functions +var ( + capType = &FuncType{builtin: "cap"} + closeType = &FuncType{builtin: "close"} + closedType = &FuncType{builtin: "closed"} + lenType = &FuncType{builtin: "len"} + makeType = &FuncType{builtin: "make"} + newType = &FuncType{builtin: "new"} + panicType = &FuncType{builtin: "panic"} + printType = &FuncType{builtin: "print"} + printlnType = &FuncType{builtin: "println"} + copyType = &FuncType{builtin: "copy"} +) + +// Two function types are identical if they have the same number of +// parameters and result values and if corresponding parameter and +// result types are identical. All "..." parameters have identical +// type. Parameter and result names are not required to match. + +func NewFuncType(in []Type, variadic bool, out []Type) *FuncType { + inMap := funcTypes + if variadic { + inMap = variadicFuncTypes + } + + outMapI := inMap.Get(in) + if outMapI == nil { + outMapI = inMap.Put(in, newTypeArrayMap()) + } + outMap := outMapI.(typeArrayMap) + + tI := outMap.Get(out) + if tI != nil { + return tI.(*FuncType) + } + + t := &FuncType{commonType{}, in, variadic, out, ""} + outMap.Put(out, t) + return t +} + +func (t *FuncType) compat(o Type, conv bool) bool { + t2, ok := o.lit().(*FuncType) + if !ok { + return false + } + if len(t.In) != len(t2.In) || t.Variadic != t2.Variadic || len(t.Out) != len(t2.Out) { + return false + } + for i := range t.In { + if !t.In[i].compat(t2.In[i], conv) { + return false + } + } + for i := range t.Out { + if !t.Out[i].compat(t2.Out[i], conv) { + return false + } + } + return true +} + +func (t *FuncType) lit() Type { return t } + +func typeListString(ts []Type, ns []*ast.Ident) string { + s := "" + for i, t := range ts { + if i > 0 { + s += ", " + } + if ns != nil && ns[i] != nil { + s += ns[i].Name + " " + } + if t == nil { + // Some places use nil types to represent errors + s += "" + } else { + s += t.String() + } + } + return s +} + +func (t *FuncType) String() string { + if t.builtin != "" { + return "built-in function " + t.builtin + } + args := typeListString(t.In, nil) + if t.Variadic { + if len(args) > 0 { + args += ", " + } + args += "..." + } + s := "func(" + args + ")" + if len(t.Out) > 0 { + s += " (" + typeListString(t.Out, nil) + ")" + } + return s +} + +func (t *FuncType) Zero() Value { return &funcV{nil} } + +type FuncDecl struct { + Type *FuncType + Name *ast.Ident // nil for function literals + // InNames will be one longer than Type.In if this function is + // variadic. + InNames []*ast.Ident + OutNames []*ast.Ident +} + +func (t *FuncDecl) String() string { + s := "func" + if t.Name != nil { + s += " " + t.Name.Name + } + s += funcTypeString(t.Type, t.InNames, t.OutNames) + return s +} + +func funcTypeString(ft *FuncType, ins []*ast.Ident, outs []*ast.Ident) string { + s := "(" + s += typeListString(ft.In, ins) + if ft.Variadic { + if len(ft.In) > 0 { + s += ", " + } + s += "..." + } + s += ")" + if len(ft.Out) > 0 { + s += " (" + typeListString(ft.Out, outs) + ")" + } + return s +} + +/* + * Interface + */ + +// TODO(austin) Interface values, types, and type compilation are +// implemented, but none of the type checking or semantics of +// interfaces are. + +type InterfaceType struct { + commonType + // TODO(austin) This should be a map from names to + // *FuncType's. We only need the sorted list for generating + // the type map key. It's detrimental for everything else. + methods []IMethod +} + +type IMethod struct { + Name string + Type *FuncType +} + +var interfaceTypes = newTypeArrayMap() + +func NewInterfaceType(methods []IMethod, embeds []*InterfaceType) *InterfaceType { + // Count methods of embedded interfaces + nMethods := len(methods) + for _, e := range embeds { + nMethods += len(e.methods) + } + + // Combine methods + allMethods := make([]IMethod, nMethods) + copy(allMethods, methods) + n := len(methods) + for _, e := range embeds { + for _, m := range e.methods { + allMethods[n] = m + n++ + } + } + + // Sort methods + sort.Sort(iMethodSorter(allMethods)) + + mts := make([]Type, len(allMethods)) + for i, m := range methods { + mts[i] = m.Type + } + tMapI := interfaceTypes.Get(mts) + if tMapI == nil { + tMapI = interfaceTypes.Put(mts, make(map[string]*InterfaceType)) + } + tMap := tMapI.(map[string]*InterfaceType) + + key := "" + for _, m := range allMethods { + key += m.Name + " " + } + + t, ok := tMap[key] + if !ok { + t = &InterfaceType{commonType{}, allMethods} + tMap[key] = t + } + return t +} + +type iMethodSorter []IMethod + +func (s iMethodSorter) Less(a, b int) bool { return s[a].Name < s[b].Name } + +func (s iMethodSorter) Swap(a, b int) { s[a], s[b] = s[b], s[a] } + +func (s iMethodSorter) Len() int { return len(s) } + +func (t *InterfaceType) compat(o Type, conv bool) bool { + t2, ok := o.lit().(*InterfaceType) + if !ok { + return false + } + if len(t.methods) != len(t2.methods) { + return false + } + for i, e := range t.methods { + e2 := t2.methods[i] + if e.Name != e2.Name || !e.Type.compat(e2.Type, conv) { + return false + } + } + return true +} + +func (t *InterfaceType) lit() Type { return t } + +func (t *InterfaceType) String() string { + // TODO(austin) Instead of showing embedded interfaces, this + // shows their methods. + s := "interface {" + for i, m := range t.methods { + if i > 0 { + s += "; " + } + s += m.Name + funcTypeString(m.Type, nil, nil) + } + return s + "}" +} + +// implementedBy tests if o implements t, returning nil, true if it does. +// Otherwise, it returns a method of t that o is missing and false. +func (t *InterfaceType) implementedBy(o Type) (*IMethod, bool) { + if len(t.methods) == 0 { + return nil, true + } + + // The methods of a named interface types are those of the + // underlying type. + if it, ok := o.lit().(*InterfaceType); ok { + o = it + } + + // XXX(Spec) Interface types: "A type implements any interface + // comprising any subset of its methods" It's unclear if + // methods must have identical or compatible types. 6g + // requires identical types. + + switch o := o.(type) { + case *NamedType: + for _, tm := range t.methods { + sm, ok := o.methods[tm.Name] + if !ok || sm.decl.Type != tm.Type { + return &tm, false + } + } + return nil, true + + case *InterfaceType: + var ti, oi int + for ti < len(t.methods) && oi < len(o.methods) { + tm, om := &t.methods[ti], &o.methods[oi] + switch { + case tm.Name == om.Name: + if tm.Type != om.Type { + return tm, false + } + ti++ + oi++ + case tm.Name > om.Name: + oi++ + default: + return tm, false + } + } + if ti < len(t.methods) { + return &t.methods[ti], false + } + return nil, true + } + + return &t.methods[0], false +} + +func (t *InterfaceType) Zero() Value { return &interfaceV{} } + +/* + * Slice + */ + +type SliceType struct { + commonType + Elem Type +} + +var sliceTypes = make(map[Type]*SliceType) + +// Two slice types are identical if they have identical element types. + +func NewSliceType(elem Type) *SliceType { + t, ok := sliceTypes[elem] + if !ok { + t = &SliceType{commonType{}, elem} + sliceTypes[elem] = t + } + return t +} + +func (t *SliceType) compat(o Type, conv bool) bool { + t2, ok := o.lit().(*SliceType) + if !ok { + return false + } + return t.Elem.compat(t2.Elem, conv) +} + +func (t *SliceType) lit() Type { return t } + +func (t *SliceType) String() string { return "[]" + t.Elem.String() } + +func (t *SliceType) Zero() Value { + // The value of an uninitialized slice is nil. The length and + // capacity of a nil slice are 0. + return &sliceV{Slice{nil, 0, 0}} +} + +/* + * Map type + */ + +type MapType struct { + commonType + Key Type + Elem Type +} + +var mapTypes = make(map[Type]map[Type]*MapType) + +func NewMapType(key Type, elem Type) *MapType { + ts, ok := mapTypes[key] + if !ok { + ts = make(map[Type]*MapType) + mapTypes[key] = ts + } + t, ok := ts[elem] + if !ok { + t = &MapType{commonType{}, key, elem} + ts[elem] = t + } + return t +} + +func (t *MapType) compat(o Type, conv bool) bool { + t2, ok := o.lit().(*MapType) + if !ok { + return false + } + return t.Elem.compat(t2.Elem, conv) && t.Key.compat(t2.Key, conv) +} + +func (t *MapType) lit() Type { return t } + +func (t *MapType) String() string { return "map[" + t.Key.String() + "] " + t.Elem.String() } + +func (t *MapType) Zero() Value { + // The value of an uninitialized map is nil. + return &mapV{nil} +} + +/* +type ChanType struct { + // TODO(austin) +} +*/ + +/* + * Named types + */ + +type Method struct { + decl *FuncDecl + fn Func +} + +type NamedType struct { + NamePos token.Pos + Name string + // Underlying type. If incomplete is true, this will be nil. + // If incomplete is false and this is still nil, then this is + // a placeholder type representing an error. + Def Type + // True while this type is being defined. + incomplete bool + methods map[string]Method +} + +// TODO(austin) This is temporarily needed by the debugger's remote +// type parser. This should only be possible with block.DefineType. +func NewNamedType(name string) *NamedType { + return &NamedType{token.NoPos, name, nil, true, make(map[string]Method)} +} + +func (t *NamedType) Pos() token.Pos { + return t.NamePos +} + +func (t *NamedType) Complete(def Type) { + if !t.incomplete { + log.Panicf("cannot complete already completed NamedType %+v", *t) + } + // We strip the name from def because multiple levels of + // naming are useless. + if ndef, ok := def.(*NamedType); ok { + def = ndef.Def + } + t.Def = def + t.incomplete = false +} + +func (t *NamedType) compat(o Type, conv bool) bool { + t2, ok := o.(*NamedType) + if ok { + if conv { + // Two named types are conversion compatible + // if their literals are conversion + // compatible. + return t.Def.compat(t2.Def, conv) + } else { + // Two named types are compatible if their + // type names originate in the same type + // declaration. + return t == t2 + } + } + // A named and an unnamed type are compatible if the + // respective type literals are compatible. + return o.compat(t.Def, conv) +} + +func (t *NamedType) lit() Type { return t.Def.lit() } + +func (t *NamedType) isBoolean() bool { return t.Def.isBoolean() } + +func (t *NamedType) isInteger() bool { return t.Def.isInteger() } + +func (t *NamedType) isFloat() bool { return t.Def.isFloat() } + +func (t *NamedType) isIdeal() bool { return false } + +func (t *NamedType) String() string { return t.Name } + +func (t *NamedType) Zero() Value { return t.Def.Zero() } + +/* + * Multi-valued type + */ + +// MultiType is a special type used for multi-valued expressions, akin +// to a tuple type. It's not generally accessible within the +// language. +type MultiType struct { + commonType + Elems []Type +} + +var multiTypes = newTypeArrayMap() + +func NewMultiType(elems []Type) *MultiType { + if t := multiTypes.Get(elems); t != nil { + return t.(*MultiType) + } + + t := &MultiType{commonType{}, elems} + multiTypes.Put(elems, t) + return t +} + +func (t *MultiType) compat(o Type, conv bool) bool { + t2, ok := o.lit().(*MultiType) + if !ok { + return false + } + if len(t.Elems) != len(t2.Elems) { + return false + } + for i := range t.Elems { + if !t.Elems[i].compat(t2.Elems[i], conv) { + return false + } + } + return true +} + +var EmptyType Type = NewMultiType([]Type{}) + +func (t *MultiType) lit() Type { return t } + +func (t *MultiType) String() string { + if len(t.Elems) == 0 { + return "" + } + return typeListString(t.Elems, nil) +} + +func (t *MultiType) Zero() Value { + res := make([]Value, len(t.Elems)) + for i, t := range t.Elems { + res[i] = t.Zero() + } + return multiV(res) +} + +/* + * Initialize the universe + */ + +func init() { + numer := big.NewInt(0xffffff) + numer.Lsh(numer, 127-23) + maxFloat32Val = new(big.Rat).SetInt(numer) + numer.SetInt64(0x1fffffffffffff) + numer.Lsh(numer, 1023-52) + maxFloat64Val = new(big.Rat).SetInt(numer) + minFloat32Val = new(big.Rat).Neg(maxFloat32Val) + minFloat64Val = new(big.Rat).Neg(maxFloat64Val) + + // To avoid portability issues all numeric types are distinct + // except byte, which is an alias for uint8. + + // Make byte an alias for the named type uint8. Type aliases + // are otherwise impossible in Go, so just hack it here. + universe.defs["byte"] = universe.defs["uint8"] + + // Built-in functions + universe.DefineConst("cap", universePos, capType, nil) + universe.DefineConst("close", universePos, closeType, nil) + universe.DefineConst("closed", universePos, closedType, nil) + universe.DefineConst("copy", universePos, copyType, nil) + universe.DefineConst("len", universePos, lenType, nil) + universe.DefineConst("make", universePos, makeType, nil) + universe.DefineConst("new", universePos, newType, nil) + universe.DefineConst("panic", universePos, panicType, nil) + universe.DefineConst("print", universePos, printType, nil) + universe.DefineConst("println", universePos, printlnType, nil) +} diff --git a/libgo/go/exp/eval/typec.go b/libgo/go/exp/eval/typec.go new file mode 100644 index 000000000..de90cf664 --- /dev/null +++ b/libgo/go/exp/eval/typec.go @@ -0,0 +1,409 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package eval + +import ( + "go/ast" + "go/token" + "log" +) + + +/* + * Type compiler + */ + +type typeCompiler struct { + *compiler + block *block + // Check to be performed after a type declaration is compiled. + // + // TODO(austin) This will probably have to change after we + // eliminate forward declarations. + lateCheck func() bool +} + +func (a *typeCompiler) compileIdent(x *ast.Ident, allowRec bool) Type { + _, _, def := a.block.Lookup(x.Name) + if def == nil { + a.diagAt(x.Pos(), "%s: undefined", x.Name) + return nil + } + switch def := def.(type) { + case *Constant: + a.diagAt(x.Pos(), "constant %v used as type", x.Name) + return nil + case *Variable: + a.diagAt(x.Pos(), "variable %v used as type", x.Name) + return nil + case *NamedType: + if !allowRec && def.incomplete { + a.diagAt(x.Pos(), "illegal recursive type") + return nil + } + if !def.incomplete && def.Def == nil { + // Placeholder type from an earlier error + return nil + } + return def + case Type: + return def + } + log.Panicf("name %s has unknown type %T", x.Name, def) + return nil +} + +func (a *typeCompiler) compileArrayType(x *ast.ArrayType, allowRec bool) Type { + // Compile element type + elem := a.compileType(x.Elt, allowRec) + + // Compile length expression + if x.Len == nil { + if elem == nil { + return nil + } + return NewSliceType(elem) + } + + if _, ok := x.Len.(*ast.Ellipsis); ok { + a.diagAt(x.Len.Pos(), "... array initailizers not implemented") + return nil + } + l, ok := a.compileArrayLen(a.block, x.Len) + if !ok { + return nil + } + if l < 0 { + a.diagAt(x.Len.Pos(), "array length must be non-negative") + return nil + } + if elem == nil { + return nil + } + + return NewArrayType(l, elem) +} + +func (a *typeCompiler) compileFields(fields *ast.FieldList, allowRec bool) ([]Type, []*ast.Ident, []token.Pos, bool) { + n := fields.NumFields() + ts := make([]Type, n) + ns := make([]*ast.Ident, n) + ps := make([]token.Pos, n) + bad := false + + if fields != nil { + i := 0 + for _, f := range fields.List { + t := a.compileType(f.Type, allowRec) + if t == nil { + bad = true + } + if f.Names == nil { + ns[i] = nil + ts[i] = t + ps[i] = f.Type.Pos() + i++ + continue + } + for _, n := range f.Names { + ns[i] = n + ts[i] = t + ps[i] = n.Pos() + i++ + } + } + } + + return ts, ns, ps, bad +} + +func (a *typeCompiler) compileStructType(x *ast.StructType, allowRec bool) Type { + ts, names, poss, bad := a.compileFields(x.Fields, allowRec) + + // XXX(Spec) The spec claims that field identifiers must be + // unique, but 6g only checks this when they are accessed. I + // think the spec is better in this regard: if I write two + // fields with the same name in the same struct type, clearly + // that's a mistake. This definition does *not* descend into + // anonymous fields, so it doesn't matter if those change. + // There's separate language in the spec about checking + // uniqueness of field names inherited from anonymous fields + // at use time. + fields := make([]StructField, len(ts)) + nameSet := make(map[string]token.Pos, len(ts)) + for i := range fields { + // Compute field name and check anonymous fields + var name string + if names[i] != nil { + name = names[i].Name + } else { + if ts[i] == nil { + continue + } + + var nt *NamedType + // [For anonymous fields,] the unqualified + // type name acts as the field identifier. + switch t := ts[i].(type) { + case *NamedType: + name = t.Name + nt = t + case *PtrType: + switch t := t.Elem.(type) { + case *NamedType: + name = t.Name + nt = t + } + } + // [An anonymous field] must be specified as a + // type name T or as a pointer to a type name + // *T, and T itself, may not be a pointer or + // interface type. + if nt == nil { + a.diagAt(poss[i], "embedded type must T or *T, where T is a named type") + bad = true + continue + } + // The check for embedded pointer types must + // be deferred because of things like + // type T *struct { T } + lateCheck := a.lateCheck + a.lateCheck = func() bool { + if _, ok := nt.lit().(*PtrType); ok { + a.diagAt(poss[i], "embedded type %v is a pointer type", nt) + return false + } + return lateCheck() + } + } + + // Check name uniqueness + if prev, ok := nameSet[name]; ok { + a.diagAt(poss[i], "field %s redeclared\n\tprevious declaration at %s", name, a.fset.Position(prev)) + bad = true + continue + } + nameSet[name] = poss[i] + + // Create field + fields[i].Name = name + fields[i].Type = ts[i] + fields[i].Anonymous = (names[i] == nil) + } + + if bad { + return nil + } + + return NewStructType(fields) +} + +func (a *typeCompiler) compilePtrType(x *ast.StarExpr) Type { + elem := a.compileType(x.X, true) + if elem == nil { + return nil + } + return NewPtrType(elem) +} + +func (a *typeCompiler) compileFuncType(x *ast.FuncType, allowRec bool) *FuncDecl { + // TODO(austin) Variadic function types + + // The types of parameters and results must be complete. + // + // TODO(austin) It's not clear they actually have to be complete. + in, inNames, _, inBad := a.compileFields(x.Params, allowRec) + out, outNames, _, outBad := a.compileFields(x.Results, allowRec) + + if inBad || outBad { + return nil + } + return &FuncDecl{NewFuncType(in, false, out), nil, inNames, outNames} +} + +func (a *typeCompiler) compileInterfaceType(x *ast.InterfaceType, allowRec bool) *InterfaceType { + ts, names, poss, bad := a.compileFields(x.Methods, allowRec) + + methods := make([]IMethod, len(ts)) + nameSet := make(map[string]token.Pos, len(ts)) + embeds := make([]*InterfaceType, len(ts)) + + var nm, ne int + for i := range ts { + if ts[i] == nil { + continue + } + + if names[i] != nil { + name := names[i].Name + methods[nm].Name = name + methods[nm].Type = ts[i].(*FuncType) + nm++ + if prev, ok := nameSet[name]; ok { + a.diagAt(poss[i], "method %s redeclared\n\tprevious declaration at %s", name, a.fset.Position(prev)) + bad = true + continue + } + nameSet[name] = poss[i] + } else { + // Embedded interface + it, ok := ts[i].lit().(*InterfaceType) + if !ok { + a.diagAt(poss[i], "embedded type must be an interface") + bad = true + continue + } + embeds[ne] = it + ne++ + for _, m := range it.methods { + if prev, ok := nameSet[m.Name]; ok { + a.diagAt(poss[i], "method %s redeclared\n\tprevious declaration at %s", m.Name, a.fset.Position(prev)) + bad = true + continue + } + nameSet[m.Name] = poss[i] + } + } + } + + if bad { + return nil + } + + methods = methods[0:nm] + embeds = embeds[0:ne] + + return NewInterfaceType(methods, embeds) +} + +func (a *typeCompiler) compileMapType(x *ast.MapType) Type { + key := a.compileType(x.Key, true) + val := a.compileType(x.Value, true) + if key == nil || val == nil { + return nil + } + // XXX(Spec) The Map types section explicitly lists all types + // that can be map keys except for function types. + switch key.lit().(type) { + case *StructType: + a.diagAt(x.Pos(), "map key cannot be a struct type") + return nil + case *ArrayType: + a.diagAt(x.Pos(), "map key cannot be an array type") + return nil + case *SliceType: + a.diagAt(x.Pos(), "map key cannot be a slice type") + return nil + } + return NewMapType(key, val) +} + +func (a *typeCompiler) compileType(x ast.Expr, allowRec bool) Type { + switch x := x.(type) { + case *ast.BadExpr: + // Error already reported by parser + a.silentErrors++ + return nil + + case *ast.Ident: + return a.compileIdent(x, allowRec) + + case *ast.ArrayType: + return a.compileArrayType(x, allowRec) + + case *ast.StructType: + return a.compileStructType(x, allowRec) + + case *ast.StarExpr: + return a.compilePtrType(x) + + case *ast.FuncType: + fd := a.compileFuncType(x, allowRec) + if fd == nil { + return nil + } + return fd.Type + + case *ast.InterfaceType: + return a.compileInterfaceType(x, allowRec) + + case *ast.MapType: + return a.compileMapType(x) + + case *ast.ChanType: + goto notimpl + + case *ast.ParenExpr: + return a.compileType(x.X, allowRec) + + case *ast.Ellipsis: + a.diagAt(x.Pos(), "illegal use of ellipsis") + return nil + } + a.diagAt(x.Pos(), "expression used as type") + return nil + +notimpl: + a.diagAt(x.Pos(), "compileType: %T not implemented", x) + return nil +} + +/* + * Type compiler interface + */ + +func noLateCheck() bool { return true } + +func (a *compiler) compileType(b *block, typ ast.Expr) Type { + tc := &typeCompiler{a, b, noLateCheck} + t := tc.compileType(typ, false) + if !tc.lateCheck() { + t = nil + } + return t +} + +func (a *compiler) compileTypeDecl(b *block, decl *ast.GenDecl) bool { + ok := true + for _, spec := range decl.Specs { + spec := spec.(*ast.TypeSpec) + // Create incomplete type for this type + nt := b.DefineType(spec.Name.Name, spec.Name.Pos(), nil) + if nt != nil { + nt.(*NamedType).incomplete = true + } + // Compile type + tc := &typeCompiler{a, b, noLateCheck} + t := tc.compileType(spec.Type, false) + if t == nil { + // Create a placeholder type + ok = false + } + // Fill incomplete type + if nt != nil { + nt.(*NamedType).Complete(t) + } + // Perform late type checking with complete type + if !tc.lateCheck() { + ok = false + if nt != nil { + // Make the type a placeholder + nt.(*NamedType).Def = nil + } + } + } + return ok +} + +func (a *compiler) compileFuncType(b *block, typ *ast.FuncType) *FuncDecl { + tc := &typeCompiler{a, b, noLateCheck} + res := tc.compileFuncType(typ, false) + if res != nil { + if !tc.lateCheck() { + res = nil + } + } + return res +} diff --git a/libgo/go/exp/eval/value.go b/libgo/go/exp/eval/value.go new file mode 100644 index 000000000..daa691897 --- /dev/null +++ b/libgo/go/exp/eval/value.go @@ -0,0 +1,586 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package eval + +import ( + "big" + "fmt" +) + +type Value interface { + String() string + // Assign copies another value into this one. It should + // assume that the other value satisfies the same specific + // value interface (BoolValue, etc.), but must not assume + // anything about its specific type. + Assign(t *Thread, o Value) +} + +type BoolValue interface { + Value + Get(*Thread) bool + Set(*Thread, bool) +} + +type UintValue interface { + Value + Get(*Thread) uint64 + Set(*Thread, uint64) +} + +type IntValue interface { + Value + Get(*Thread) int64 + Set(*Thread, int64) +} + +// TODO(austin) IdealIntValue and IdealFloatValue should not exist +// because ideals are not l-values. +type IdealIntValue interface { + Value + Get() *big.Int +} + +type FloatValue interface { + Value + Get(*Thread) float64 + Set(*Thread, float64) +} + +type IdealFloatValue interface { + Value + Get() *big.Rat +} + +type StringValue interface { + Value + Get(*Thread) string + Set(*Thread, string) +} + +type ArrayValue interface { + Value + // TODO(austin) Get() is here for uniformity, but is + // completely useless. If a lot of other types have similarly + // useless Get methods, just special-case these uses. + Get(*Thread) ArrayValue + Elem(*Thread, int64) Value + // Sub returns an ArrayValue backed by the same array that + // starts from element i and has length len. + Sub(i int64, len int64) ArrayValue +} + +type StructValue interface { + Value + // TODO(austin) This is another useless Get() + Get(*Thread) StructValue + Field(*Thread, int) Value +} + +type PtrValue interface { + Value + Get(*Thread) Value + Set(*Thread, Value) +} + +type Func interface { + NewFrame() *Frame + Call(*Thread) +} + +type FuncValue interface { + Value + Get(*Thread) Func + Set(*Thread, Func) +} + +type Interface struct { + Type Type + Value Value +} + +type InterfaceValue interface { + Value + Get(*Thread) Interface + Set(*Thread, Interface) +} + +type Slice struct { + Base ArrayValue + Len, Cap int64 +} + +type SliceValue interface { + Value + Get(*Thread) Slice + Set(*Thread, Slice) +} + +type Map interface { + Len(*Thread) int64 + // Retrieve an element from the map, returning nil if it does + // not exist. + Elem(t *Thread, key interface{}) Value + // Set an entry in the map. If val is nil, delete the entry. + SetElem(t *Thread, key interface{}, val Value) + // TODO(austin) Perhaps there should be an iterator interface instead. + Iter(func(key interface{}, val Value) bool) +} + +type MapValue interface { + Value + Get(*Thread) Map + Set(*Thread, Map) +} + +/* + * Bool + */ + +type boolV bool + +func (v *boolV) String() string { return fmt.Sprint(*v) } + +func (v *boolV) Assign(t *Thread, o Value) { *v = boolV(o.(BoolValue).Get(t)) } + +func (v *boolV) Get(*Thread) bool { return bool(*v) } + +func (v *boolV) Set(t *Thread, x bool) { *v = boolV(x) } + +/* + * Uint + */ + +type uint8V uint8 + +func (v *uint8V) String() string { return fmt.Sprint(*v) } + +func (v *uint8V) Assign(t *Thread, o Value) { *v = uint8V(o.(UintValue).Get(t)) } + +func (v *uint8V) Get(*Thread) uint64 { return uint64(*v) } + +func (v *uint8V) Set(t *Thread, x uint64) { *v = uint8V(x) } + +type uint16V uint16 + +func (v *uint16V) String() string { return fmt.Sprint(*v) } + +func (v *uint16V) Assign(t *Thread, o Value) { *v = uint16V(o.(UintValue).Get(t)) } + +func (v *uint16V) Get(*Thread) uint64 { return uint64(*v) } + +func (v *uint16V) Set(t *Thread, x uint64) { *v = uint16V(x) } + +type uint32V uint32 + +func (v *uint32V) String() string { return fmt.Sprint(*v) } + +func (v *uint32V) Assign(t *Thread, o Value) { *v = uint32V(o.(UintValue).Get(t)) } + +func (v *uint32V) Get(*Thread) uint64 { return uint64(*v) } + +func (v *uint32V) Set(t *Thread, x uint64) { *v = uint32V(x) } + +type uint64V uint64 + +func (v *uint64V) String() string { return fmt.Sprint(*v) } + +func (v *uint64V) Assign(t *Thread, o Value) { *v = uint64V(o.(UintValue).Get(t)) } + +func (v *uint64V) Get(*Thread) uint64 { return uint64(*v) } + +func (v *uint64V) Set(t *Thread, x uint64) { *v = uint64V(x) } + +type uintV uint + +func (v *uintV) String() string { return fmt.Sprint(*v) } + +func (v *uintV) Assign(t *Thread, o Value) { *v = uintV(o.(UintValue).Get(t)) } + +func (v *uintV) Get(*Thread) uint64 { return uint64(*v) } + +func (v *uintV) Set(t *Thread, x uint64) { *v = uintV(x) } + +type uintptrV uintptr + +func (v *uintptrV) String() string { return fmt.Sprint(*v) } + +func (v *uintptrV) Assign(t *Thread, o Value) { *v = uintptrV(o.(UintValue).Get(t)) } + +func (v *uintptrV) Get(*Thread) uint64 { return uint64(*v) } + +func (v *uintptrV) Set(t *Thread, x uint64) { *v = uintptrV(x) } + +/* + * Int + */ + +type int8V int8 + +func (v *int8V) String() string { return fmt.Sprint(*v) } + +func (v *int8V) Assign(t *Thread, o Value) { *v = int8V(o.(IntValue).Get(t)) } + +func (v *int8V) Get(*Thread) int64 { return int64(*v) } + +func (v *int8V) Set(t *Thread, x int64) { *v = int8V(x) } + +type int16V int16 + +func (v *int16V) String() string { return fmt.Sprint(*v) } + +func (v *int16V) Assign(t *Thread, o Value) { *v = int16V(o.(IntValue).Get(t)) } + +func (v *int16V) Get(*Thread) int64 { return int64(*v) } + +func (v *int16V) Set(t *Thread, x int64) { *v = int16V(x) } + +type int32V int32 + +func (v *int32V) String() string { return fmt.Sprint(*v) } + +func (v *int32V) Assign(t *Thread, o Value) { *v = int32V(o.(IntValue).Get(t)) } + +func (v *int32V) Get(*Thread) int64 { return int64(*v) } + +func (v *int32V) Set(t *Thread, x int64) { *v = int32V(x) } + +type int64V int64 + +func (v *int64V) String() string { return fmt.Sprint(*v) } + +func (v *int64V) Assign(t *Thread, o Value) { *v = int64V(o.(IntValue).Get(t)) } + +func (v *int64V) Get(*Thread) int64 { return int64(*v) } + +func (v *int64V) Set(t *Thread, x int64) { *v = int64V(x) } + +type intV int + +func (v *intV) String() string { return fmt.Sprint(*v) } + +func (v *intV) Assign(t *Thread, o Value) { *v = intV(o.(IntValue).Get(t)) } + +func (v *intV) Get(*Thread) int64 { return int64(*v) } + +func (v *intV) Set(t *Thread, x int64) { *v = intV(x) } + +/* + * Ideal int + */ + +type idealIntV struct { + V *big.Int +} + +func (v *idealIntV) String() string { return v.V.String() } + +func (v *idealIntV) Assign(t *Thread, o Value) { + v.V = o.(IdealIntValue).Get() +} + +func (v *idealIntV) Get() *big.Int { return v.V } + +/* + * Float + */ + +type float32V float32 + +func (v *float32V) String() string { return fmt.Sprint(*v) } + +func (v *float32V) Assign(t *Thread, o Value) { *v = float32V(o.(FloatValue).Get(t)) } + +func (v *float32V) Get(*Thread) float64 { return float64(*v) } + +func (v *float32V) Set(t *Thread, x float64) { *v = float32V(x) } + +type float64V float64 + +func (v *float64V) String() string { return fmt.Sprint(*v) } + +func (v *float64V) Assign(t *Thread, o Value) { *v = float64V(o.(FloatValue).Get(t)) } + +func (v *float64V) Get(*Thread) float64 { return float64(*v) } + +func (v *float64V) Set(t *Thread, x float64) { *v = float64V(x) } + +/* + * Ideal float + */ + +type idealFloatV struct { + V *big.Rat +} + +func (v *idealFloatV) String() string { return v.V.FloatString(6) } + +func (v *idealFloatV) Assign(t *Thread, o Value) { + v.V = o.(IdealFloatValue).Get() +} + +func (v *idealFloatV) Get() *big.Rat { return v.V } + +/* + * String + */ + +type stringV string + +func (v *stringV) String() string { return fmt.Sprint(*v) } + +func (v *stringV) Assign(t *Thread, o Value) { *v = stringV(o.(StringValue).Get(t)) } + +func (v *stringV) Get(*Thread) string { return string(*v) } + +func (v *stringV) Set(t *Thread, x string) { *v = stringV(x) } + +/* + * Array + */ + +type arrayV []Value + +func (v *arrayV) String() string { + res := "{" + for i, e := range *v { + if i > 0 { + res += ", " + } + res += e.String() + } + return res + "}" +} + +func (v *arrayV) Assign(t *Thread, o Value) { + oa := o.(ArrayValue) + l := int64(len(*v)) + for i := int64(0); i < l; i++ { + (*v)[i].Assign(t, oa.Elem(t, i)) + } +} + +func (v *arrayV) Get(*Thread) ArrayValue { return v } + +func (v *arrayV) Elem(t *Thread, i int64) Value { + return (*v)[i] +} + +func (v *arrayV) Sub(i int64, len int64) ArrayValue { + res := (*v)[i : i+len] + return &res +} + +/* + * Struct + */ + +type structV []Value + +// TODO(austin) Should these methods (and arrayV's) be on structV +// instead of *structV? +func (v *structV) String() string { + res := "{" + for i, v := range *v { + if i > 0 { + res += ", " + } + res += v.String() + } + return res + "}" +} + +func (v *structV) Assign(t *Thread, o Value) { + oa := o.(StructValue) + l := len(*v) + for i := 0; i < l; i++ { + (*v)[i].Assign(t, oa.Field(t, i)) + } +} + +func (v *structV) Get(*Thread) StructValue { return v } + +func (v *structV) Field(t *Thread, i int) Value { + return (*v)[i] +} + +/* + * Pointer + */ + +type ptrV struct { + // nil if the pointer is nil + target Value +} + +func (v *ptrV) String() string { + if v.target == nil { + return "" + } + return "&" + v.target.String() +} + +func (v *ptrV) Assign(t *Thread, o Value) { v.target = o.(PtrValue).Get(t) } + +func (v *ptrV) Get(*Thread) Value { return v.target } + +func (v *ptrV) Set(t *Thread, x Value) { v.target = x } + +/* + * Functions + */ + +type funcV struct { + target Func +} + +func (v *funcV) String() string { + // TODO(austin) Rob wants to see the definition + return "func {...}" +} + +func (v *funcV) Assign(t *Thread, o Value) { v.target = o.(FuncValue).Get(t) } + +func (v *funcV) Get(*Thread) Func { return v.target } + +func (v *funcV) Set(t *Thread, x Func) { v.target = x } + +/* + * Interfaces + */ + +type interfaceV struct { + Interface +} + +func (v *interfaceV) String() string { + if v.Type == nil || v.Value == nil { + return "" + } + return v.Value.String() +} + +func (v *interfaceV) Assign(t *Thread, o Value) { + v.Interface = o.(InterfaceValue).Get(t) +} + +func (v *interfaceV) Get(*Thread) Interface { return v.Interface } + +func (v *interfaceV) Set(t *Thread, x Interface) { + v.Interface = x +} + +/* + * Slices + */ + +type sliceV struct { + Slice +} + +func (v *sliceV) String() string { + if v.Base == nil { + return "" + } + return v.Base.Sub(0, v.Len).String() +} + +func (v *sliceV) Assign(t *Thread, o Value) { v.Slice = o.(SliceValue).Get(t) } + +func (v *sliceV) Get(*Thread) Slice { return v.Slice } + +func (v *sliceV) Set(t *Thread, x Slice) { v.Slice = x } + +/* + * Maps + */ + +type mapV struct { + target Map +} + +func (v *mapV) String() string { + if v.target == nil { + return "" + } + res := "map[" + i := 0 + v.target.Iter(func(key interface{}, val Value) bool { + if i > 0 { + res += ", " + } + i++ + res += fmt.Sprint(key) + ":" + val.String() + return true + }) + return res + "]" +} + +func (v *mapV) Assign(t *Thread, o Value) { v.target = o.(MapValue).Get(t) } + +func (v *mapV) Get(*Thread) Map { return v.target } + +func (v *mapV) Set(t *Thread, x Map) { v.target = x } + +type evalMap map[interface{}]Value + +func (m evalMap) Len(t *Thread) int64 { return int64(len(m)) } + +func (m evalMap) Elem(t *Thread, key interface{}) Value { + return m[key] +} + +func (m evalMap) SetElem(t *Thread, key interface{}, val Value) { + if val == nil { + m[key] = nil, false + } else { + m[key] = val + } +} + +func (m evalMap) Iter(cb func(key interface{}, val Value) bool) { + for k, v := range m { + if !cb(k, v) { + break + } + } +} + +/* + * Multi-values + */ + +type multiV []Value + +func (v multiV) String() string { + res := "(" + for i, v := range v { + if i > 0 { + res += ", " + } + res += v.String() + } + return res + ")" +} + +func (v multiV) Assign(t *Thread, o Value) { + omv := o.(multiV) + for i := range v { + v[i].Assign(t, omv[i]) + } +} + +/* + * Universal constants + */ + +func init() { + s := universe + + true := boolV(true) + s.DefineConst("true", universePos, BoolType, &true) + false := boolV(false) + s.DefineConst("false", universePos, BoolType, &false) +} diff --git a/libgo/go/exp/eval/world.go b/libgo/go/exp/eval/world.go new file mode 100644 index 000000000..02d18bd79 --- /dev/null +++ b/libgo/go/exp/eval/world.go @@ -0,0 +1,188 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This package is the beginning of an interpreter for Go. +// It can run simple Go programs but does not implement +// interface values or packages. +package eval + +import ( + "go/ast" + "go/parser" + "go/scanner" + "go/token" + "os" +) + +type World struct { + scope *Scope + frame *Frame +} + +func NewWorld() *World { + w := new(World) + w.scope = universe.ChildScope() + w.scope.global = true // this block's vars allocate directly + return w +} + +type Code interface { + // The type of the value Run returns, or nil if Run returns nil. + Type() Type + + // Run runs the code; if the code is a single expression + // with a value, it returns the value; otherwise it returns nil. + Run() (Value, os.Error) +} + +type stmtCode struct { + w *World + code code +} + +func (w *World) CompileStmtList(fset *token.FileSet, stmts []ast.Stmt) (Code, os.Error) { + if len(stmts) == 1 { + if s, ok := stmts[0].(*ast.ExprStmt); ok { + return w.CompileExpr(fset, s.X) + } + } + errors := new(scanner.ErrorVector) + cc := &compiler{fset, errors, 0, 0} + cb := newCodeBuf() + fc := &funcCompiler{ + compiler: cc, + fnType: nil, + outVarsNamed: false, + codeBuf: cb, + flow: newFlowBuf(cb), + labels: make(map[string]*label), + } + bc := &blockCompiler{ + funcCompiler: fc, + block: w.scope.block, + } + nerr := cc.numError() + for _, stmt := range stmts { + bc.compileStmt(stmt) + } + fc.checkLabels() + if nerr != cc.numError() { + return nil, errors.GetError(scanner.Sorted) + } + return &stmtCode{w, fc.get()}, nil +} + +func (w *World) CompileDeclList(fset *token.FileSet, decls []ast.Decl) (Code, os.Error) { + stmts := make([]ast.Stmt, len(decls)) + for i, d := range decls { + stmts[i] = &ast.DeclStmt{d} + } + return w.CompileStmtList(fset, stmts) +} + +func (s *stmtCode) Type() Type { return nil } + +func (s *stmtCode) Run() (Value, os.Error) { + t := new(Thread) + t.f = s.w.scope.NewFrame(nil) + return nil, t.Try(func(t *Thread) { s.code.exec(t) }) +} + +type exprCode struct { + w *World + e *expr + eval func(Value, *Thread) +} + +func (w *World) CompileExpr(fset *token.FileSet, e ast.Expr) (Code, os.Error) { + errors := new(scanner.ErrorVector) + cc := &compiler{fset, errors, 0, 0} + + ec := cc.compileExpr(w.scope.block, false, e) + if ec == nil { + return nil, errors.GetError(scanner.Sorted) + } + var eval func(Value, *Thread) + switch t := ec.t.(type) { + case *idealIntType: + // nothing + case *idealFloatType: + // nothing + default: + if tm, ok := t.(*MultiType); ok && len(tm.Elems) == 0 { + return &stmtCode{w, code{ec.exec}}, nil + } + eval = genAssign(ec.t, ec) + } + return &exprCode{w, ec, eval}, nil +} + +func (e *exprCode) Type() Type { return e.e.t } + +func (e *exprCode) Run() (Value, os.Error) { + t := new(Thread) + t.f = e.w.scope.NewFrame(nil) + switch e.e.t.(type) { + case *idealIntType: + return &idealIntV{e.e.asIdealInt()()}, nil + case *idealFloatType: + return &idealFloatV{e.e.asIdealFloat()()}, nil + } + v := e.e.t.Zero() + eval := e.eval + err := t.Try(func(t *Thread) { eval(v, t) }) + return v, err +} + +func (w *World) Compile(fset *token.FileSet, text string) (Code, os.Error) { + stmts, err := parser.ParseStmtList(fset, "input", text) + if err == nil { + return w.CompileStmtList(fset, stmts) + } + + // Otherwise try as DeclList. + decls, err1 := parser.ParseDeclList(fset, "input", text) + if err1 == nil { + return w.CompileDeclList(fset, decls) + } + + // Have to pick an error. + // Parsing as statement list admits more forms, + // its error is more likely to be useful. + return nil, err +} + +type RedefinitionError struct { + Name string + Prev Def +} + +func (e *RedefinitionError) String() string { + res := "identifier " + e.Name + " redeclared" + pos := e.Prev.Pos() + if pos.IsValid() { + // TODO: fix this - currently this code is not reached by the tests + // need to get a file set (fset) from somewhere + //res += "; previous declaration at " + fset.Position(pos).String() + panic(0) + } + return res +} + +func (w *World) DefineConst(name string, t Type, val Value) os.Error { + _, prev := w.scope.DefineConst(name, token.NoPos, t, val) + if prev != nil { + return &RedefinitionError{name, prev} + } + return nil +} + +func (w *World) DefineVar(name string, t Type, val Value) os.Error { + v, prev := w.scope.DefineVar(name, token.NoPos, t) + if prev != nil { + return &RedefinitionError{name, prev} + } + v.Init = val + return nil +} diff --git a/libgo/go/exp/ogle/abort.go b/libgo/go/exp/ogle/abort.go new file mode 100644 index 000000000..311a7b38e --- /dev/null +++ b/libgo/go/exp/ogle/abort.go @@ -0,0 +1,35 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ogle + +import ( + "os" + "runtime" +) + +// An aborter aborts the thread's current computation, usually +// passing the error to a waiting thread. +type aborter interface { + Abort(err os.Error) +} + +type ogleAborter chan os.Error + +func (a ogleAborter) Abort(err os.Error) { + a <- err + runtime.Goexit() +} + +// try executes a computation; if the computation Aborts, try returns +// the error passed to abort. +func try(f func(a aborter)) os.Error { + a := make(ogleAborter) + go func() { + f(a) + a <- nil + }() + err := <-a + return err +} diff --git a/libgo/go/exp/ogle/arch.go b/libgo/go/exp/ogle/arch.go new file mode 100644 index 000000000..52b1c9757 --- /dev/null +++ b/libgo/go/exp/ogle/arch.go @@ -0,0 +1,125 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ogle + +import ( + "debug/proc" + "math" +) + +type Arch interface { + // ToWord converts an array of up to 8 bytes in memory order + // to a word. + ToWord(data []byte) proc.Word + // FromWord converts a word to an array of up to 8 bytes in + // memory order. + FromWord(v proc.Word, out []byte) + // ToFloat32 converts a word to a float. The order of this + // word will be the order returned by ToWord on the memory + // representation of a float, and thus may require reversing. + ToFloat32(bits uint32) float32 + // FromFloat32 converts a float to a word. This should return + // a word that can be passed to FromWord to get the memory + // representation of a float on this architecture. + FromFloat32(f float32) uint32 + // ToFloat64 is to float64 as ToFloat32 is to float32. + ToFloat64(bits uint64) float64 + // FromFloat64 is to float64 as FromFloat32 is to float32. + FromFloat64(f float64) uint64 + + // IntSize returns the number of bytes in an 'int'. + IntSize() int + // PtrSize returns the number of bytes in a 'uintptr'. + PtrSize() int + // FloatSize returns the number of bytes in a 'float'. + FloatSize() int + // Align rounds offset up to the appropriate offset for a + // basic type with the given width. + Align(offset, width int) int + + // G returns the current G pointer. + G(regs proc.Regs) proc.Word + + // ClosureSize returns the number of bytes expected by + // ParseClosure. + ClosureSize() int + // ParseClosure takes ClosureSize bytes read from a return PC + // in a remote process, determines if the code is a closure, + // and returns the frame size of the closure if it is. + ParseClosure(data []byte) (frame int, ok bool) +} + +type ArchLSB struct{} + +func (ArchLSB) ToWord(data []byte) proc.Word { + var v proc.Word + for i, b := range data { + v |= proc.Word(b) << (uint(i) * 8) + } + return v +} + +func (ArchLSB) FromWord(v proc.Word, out []byte) { + for i := range out { + out[i] = byte(v) + v >>= 8 + } +} + +func (ArchLSB) ToFloat32(bits uint32) float32 { + // TODO(austin) Do these definitions depend on my current + // architecture? + return math.Float32frombits(bits) +} + +func (ArchLSB) FromFloat32(f float32) uint32 { return math.Float32bits(f) } + +func (ArchLSB) ToFloat64(bits uint64) float64 { return math.Float64frombits(bits) } + +func (ArchLSB) FromFloat64(f float64) uint64 { return math.Float64bits(f) } + +type ArchAlignedMultiple struct{} + +func (ArchAlignedMultiple) Align(offset, width int) int { + return ((offset - 1) | (width - 1)) + 1 +} + +type amd64 struct { + ArchLSB + ArchAlignedMultiple + gReg int +} + +func (a *amd64) IntSize() int { return 4 } + +func (a *amd64) PtrSize() int { return 8 } + +func (a *amd64) FloatSize() int { return 4 } + +func (a *amd64) G(regs proc.Regs) proc.Word { + // See src/pkg/runtime/mkasmh + if a.gReg == -1 { + ns := regs.Names() + for i, n := range ns { + if n == "r15" { + a.gReg = i + break + } + } + } + + return regs.Get(a.gReg) +} + +func (a *amd64) ClosureSize() int { return 8 } + +func (a *amd64) ParseClosure(data []byte) (int, bool) { + if data[0] == 0x48 && data[1] == 0x81 && data[2] == 0xc4 && data[7] == 0xc3 { + return int(a.ToWord(data[3:7]) + 8), true + } + return 0, false +} + +var Amd64 = &amd64{gReg: -1} diff --git a/libgo/go/exp/ogle/cmd.go b/libgo/go/exp/ogle/cmd.go new file mode 100644 index 000000000..4f67032d0 --- /dev/null +++ b/libgo/go/exp/ogle/cmd.go @@ -0,0 +1,373 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Ogle is the beginning of a debugger for Go. +package ogle + +import ( + "bufio" + "debug/elf" + "debug/proc" + "exp/eval" + "fmt" + "go/scanner" + "go/token" + "os" + "strconv" + "strings" +) + +var fset = token.NewFileSet() +var world *eval.World +var curProc *Process + +func Main() { + world = eval.NewWorld() + defineFuncs() + r := bufio.NewReader(os.Stdin) + for { + print("; ") + line, err := r.ReadSlice('\n') + if err != nil { + break + } + + // Try line as a command + cmd, rest := getCmd(line) + if cmd != nil { + err := cmd.handler(rest) + if err != nil { + scanner.PrintError(os.Stderr, err) + } + continue + } + + // Try line as code + code, err := world.Compile(fset, string(line)) + if err != nil { + scanner.PrintError(os.Stderr, err) + continue + } + v, err := code.Run() + if err != nil { + fmt.Fprintf(os.Stderr, err.String()) + continue + } + if v != nil { + println(v.String()) + } + } +} + +// newScanner creates a new scanner that scans that given input bytes. +func newScanner(input []byte) (*scanner.Scanner, *scanner.ErrorVector) { + sc := new(scanner.Scanner) + ev := new(scanner.ErrorVector) + file := fset.AddFile("input", fset.Base(), len(input)) + sc.Init(file, input, ev, 0) + return sc, ev +} + +/* + * Commands + */ + +// A UsageError occurs when a command is called with illegal arguments. +type UsageError string + +func (e UsageError) String() string { return string(e) } + +// A cmd represents a single command with a handler. +type cmd struct { + cmd string + handler func([]byte) os.Error +} + +var cmds = []cmd{ + {"load", cmdLoad}, + {"bt", cmdBt}, +} + +// getCmd attempts to parse an input line as a registered command. If +// successful, it returns the command and the bytes remaining after +// the command, which should be passed to the command. +func getCmd(line []byte) (*cmd, []byte) { + sc, _ := newScanner(line) + pos, tok, lit := sc.Scan() + if sc.ErrorCount != 0 || tok != token.IDENT { + return nil, nil + } + + slit := string(lit) + for i := range cmds { + if cmds[i].cmd == slit { + return &cmds[i], line[fset.Position(pos).Offset+len(lit):] + } + } + return nil, nil +} + +// cmdLoad starts or attaches to a process. Its form is similar to +// import: +// +// load [sym] "path" [;] +// +// sym specifies the name to give to the process. If not given, the +// name is derived from the path of the process. If ".", then the +// packages from the remote process are defined into the current +// namespace. If given, this symbol is defined as a package +// containing the process' packages. +// +// path gives the path of the process to start or attach to. If it is +// "pid:", then attach to the given PID. Otherwise, treat it as +// a file path and space-separated arguments and start a new process. +// +// load always sets the current process to the loaded process. +func cmdLoad(args []byte) os.Error { + ident, path, err := parseLoad(args) + if err != nil { + return err + } + if curProc != nil { + return UsageError("multiple processes not implemented") + } + if ident != "." { + return UsageError("process identifiers not implemented") + } + + // Parse argument and start or attach to process + var fname string + var tproc proc.Process + if len(path) >= 4 && path[0:4] == "pid:" { + pid, err := strconv.Atoi(path[4:]) + if err != nil { + return err + } + fname, err = os.Readlink(fmt.Sprintf("/proc/%d/exe", pid)) + if err != nil { + return err + } + tproc, err = proc.Attach(pid) + if err != nil { + return err + } + println("Attached to", pid) + } else { + parts := strings.Split(path, " ", -1) + if len(parts) == 0 { + fname = "" + } else { + fname = parts[0] + } + tproc, err = proc.ForkExec(fname, parts, os.Environ(), "", []*os.File{os.Stdin, os.Stdout, os.Stderr}) + if err != nil { + return err + } + println("Started", path) + // TODO(austin) If we fail after this point, kill tproc + // before detaching. + } + + // Get symbols + f, err := os.Open(fname, os.O_RDONLY, 0) + if err != nil { + tproc.Detach() + return err + } + defer f.Close() + elf, err := elf.NewFile(f) + if err != nil { + tproc.Detach() + return err + } + curProc, err = NewProcessElf(tproc, elf) + if err != nil { + tproc.Detach() + return err + } + + // Prepare new process + curProc.OnGoroutineCreate().AddHandler(EventPrint) + curProc.OnGoroutineExit().AddHandler(EventPrint) + + err = curProc.populateWorld(world) + if err != nil { + tproc.Detach() + return err + } + + return nil +} + +func parseLoad(args []byte) (ident string, path string, err os.Error) { + err = UsageError("Usage: load [sym] \"path\"") + sc, ev := newScanner(args) + + var toks [4]token.Token + var lits [4][]byte + for i := range toks { + _, toks[i], lits[i] = sc.Scan() + } + if sc.ErrorCount != 0 { + err = ev.GetError(scanner.NoMultiples) + return + } + + i := 0 + switch toks[i] { + case token.PERIOD, token.IDENT: + ident = string(lits[i]) + i++ + } + + if toks[i] != token.STRING { + return + } + path, uerr := strconv.Unquote(string(lits[i])) + if uerr != nil { + err = uerr + return + } + i++ + + if toks[i] == token.SEMICOLON { + i++ + } + if toks[i] != token.EOF { + return + } + + return ident, path, nil +} + +// cmdBt prints a backtrace for the current goroutine. It takes no +// arguments. +func cmdBt(args []byte) os.Error { + err := parseNoArgs(args, "Usage: bt") + if err != nil { + return err + } + + if curProc == nil || curProc.curGoroutine == nil { + return NoCurrentGoroutine{} + } + + f := curProc.curGoroutine.frame + if f == nil { + fmt.Println("No frames on stack") + return nil + } + + for f.Inner() != nil { + f = f.Inner() + } + + for i := 0; i < 100; i++ { + if f == curProc.curGoroutine.frame { + fmt.Printf("=> ") + } else { + fmt.Printf(" ") + } + fmt.Printf("%8x %v\n", f.pc, f) + f, err = f.Outer() + if err != nil { + return err + } + if f == nil { + return nil + } + } + + fmt.Println("...") + return nil +} + +func parseNoArgs(args []byte, usage string) os.Error { + sc, ev := newScanner(args) + _, tok, _ := sc.Scan() + if sc.ErrorCount != 0 { + return ev.GetError(scanner.NoMultiples) + } + if tok != token.EOF { + return UsageError(usage) + } + return nil +} + +/* + * Functions + */ + +// defineFuncs populates world with the built-in functions. +func defineFuncs() { + t, v := eval.FuncFromNativeTyped(fnOut, fnOutSig) + world.DefineConst("Out", t, v) + t, v = eval.FuncFromNativeTyped(fnContWait, fnContWaitSig) + world.DefineConst("ContWait", t, v) + t, v = eval.FuncFromNativeTyped(fnBpSet, fnBpSetSig) + world.DefineConst("BpSet", t, v) +} + +// printCurFrame prints the current stack frame, as it would appear in +// a backtrace. +func printCurFrame() { + if curProc == nil || curProc.curGoroutine == nil { + return + } + f := curProc.curGoroutine.frame + if f == nil { + return + } + fmt.Printf("=> %8x %v\n", f.pc, f) +} + +// fnOut moves the current frame to the caller of the current frame. +func fnOutSig() {} +func fnOut(t *eval.Thread, args []eval.Value, res []eval.Value) { + if curProc == nil { + t.Abort(NoCurrentGoroutine{}) + } + err := curProc.Out() + if err != nil { + t.Abort(err) + } + // TODO(austin) Only in the command form + printCurFrame() +} + +// fnContWait continues the current process and waits for a stopping event. +func fnContWaitSig() {} +func fnContWait(t *eval.Thread, args []eval.Value, res []eval.Value) { + if curProc == nil { + t.Abort(NoCurrentGoroutine{}) + } + err := curProc.ContWait() + if err != nil { + t.Abort(err) + } + // TODO(austin) Only in the command form + ev := curProc.Event() + if ev != nil { + fmt.Printf("%v\n", ev) + } + printCurFrame() +} + +// fnBpSet sets a breakpoint at the entry to the named function. +func fnBpSetSig(string) {} +func fnBpSet(t *eval.Thread, args []eval.Value, res []eval.Value) { + // TODO(austin) This probably shouldn't take a symbol name. + // Perhaps it should take an interface that provides PC's. + // Functions and instructions can implement that interface and + // we can have something to translate file:line pairs. + if curProc == nil { + t.Abort(NoCurrentGoroutine{}) + } + name := args[0].(eval.StringValue).Get(t) + fn := curProc.syms.LookupFunc(name) + if fn == nil { + t.Abort(UsageError("no such function " + name)) + } + curProc.OnBreakpoint(proc.Word(fn.Entry)).AddHandler(EventStop) +} diff --git a/libgo/go/exp/ogle/event.go b/libgo/go/exp/ogle/event.go new file mode 100644 index 000000000..d7092ded3 --- /dev/null +++ b/libgo/go/exp/ogle/event.go @@ -0,0 +1,280 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ogle + +import ( + "debug/proc" + "fmt" + "os" +) + +/* + * Hooks and events + */ + +// An EventHandler is a function that takes an event and returns a +// response to that event and possibly an error. If an event handler +// returns an error, the process stops and no other handlers for that +// event are executed. +type EventHandler func(e Event) (EventAction, os.Error) + +// An EventAction is an event handler's response to an event. If all +// of an event's handlers execute without returning errors, their +// results are combined as follows: If any handler returned +// EAContinue, then the process resumes (without returning from +// WaitStop); otherwise, if any handler returned EAStop, the process +// remains stopped; otherwise, if all handlers returned EADefault, the +// process resumes. A handler may return EARemoveSelf bit-wise or'd +// with any other action to indicate that the handler should be +// removed from the hook. +type EventAction int + +const ( + EARemoveSelf EventAction = 0x100 + EADefault EventAction = iota + EAStop + EAContinue +) + +// A EventHook allows event handlers to be added and removed. +type EventHook interface { + AddHandler(EventHandler) + RemoveHandler(EventHandler) + NumHandler() int + handle(e Event) (EventAction, os.Error) + String() string +} + +// EventHook is almost, but not quite, suitable for user-defined +// events. If we want user-defined events, make EventHook a struct, +// special-case adding and removing handlers in breakpoint hooks, and +// provide a public interface for posting events to hooks. + +type Event interface { + Process() *Process + Goroutine() *Goroutine + String() string +} + +type commonHook struct { + // Head of handler chain + head *handler + // Number of non-internal handlers + len int +} + +type handler struct { + eh EventHandler + // True if this handler must be run before user-defined + // handlers in order to ensure correctness. + internal bool + // True if this handler has been removed from the chain. + removed bool + next *handler +} + +func (h *commonHook) AddHandler(eh EventHandler) { + h.addHandler(eh, false) +} + +func (h *commonHook) addHandler(eh EventHandler, internal bool) { + // Ensure uniqueness of handlers + h.RemoveHandler(eh) + + if !internal { + h.len++ + } + // Add internal handlers to the beginning + if internal || h.head == nil { + h.head = &handler{eh, internal, false, h.head} + return + } + // Add handler after internal handlers + // TODO(austin) This should probably go on the end instead + prev := h.head + for prev.next != nil && prev.internal { + prev = prev.next + } + prev.next = &handler{eh, internal, false, prev.next} +} + +func (h *commonHook) RemoveHandler(eh EventHandler) { + plink := &h.head + for l := *plink; l != nil; plink, l = &l.next, l.next { + if l.eh == eh { + if !l.internal { + h.len-- + } + l.removed = true + *plink = l.next + break + } + } +} + +func (h *commonHook) NumHandler() int { return h.len } + +func (h *commonHook) handle(e Event) (EventAction, os.Error) { + action := EADefault + plink := &h.head + for l := *plink; l != nil; plink, l = &l.next, l.next { + if l.removed { + continue + } + a, err := l.eh(e) + if a&EARemoveSelf == EARemoveSelf { + if !l.internal { + h.len-- + } + l.removed = true + *plink = l.next + a &^= EARemoveSelf + } + if err != nil { + return EAStop, err + } + if a > action { + action = a + } + } + return action, nil +} + +type commonEvent struct { + // The process of this event + p *Process + // The goroutine of this event. + t *Goroutine +} + +func (e *commonEvent) Process() *Process { return e.p } + +func (e *commonEvent) Goroutine() *Goroutine { return e.t } + +/* + * Standard event handlers + */ + +// EventPrint is a standard event handler that prints events as they +// occur. It will not cause the process to stop. +func EventPrint(ev Event) (EventAction, os.Error) { + // TODO(austin) Include process name here? + fmt.Fprintf(os.Stderr, "*** %v\n", ev.String()) + return EADefault, nil +} + +// EventStop is a standard event handler that causes the process to stop. +func EventStop(ev Event) (EventAction, os.Error) { + return EAStop, nil +} + +/* + * Breakpoints + */ + +type breakpointHook struct { + commonHook + p *Process + pc proc.Word +} + +// A Breakpoint event occurs when a process reaches a particular +// program counter. When this event is handled, the current goroutine +// will be the goroutine that reached the program counter. +type Breakpoint struct { + commonEvent + osThread proc.Thread + pc proc.Word +} + +func (h *breakpointHook) AddHandler(eh EventHandler) { + h.addHandler(eh, false) +} + +func (h *breakpointHook) addHandler(eh EventHandler, internal bool) { + // We register breakpoint events lazily to avoid holding + // references to breakpoints without handlers. Be sure to use + // the "canonical" breakpoint if there is one. + if cur, ok := h.p.breakpointHooks[h.pc]; ok { + h = cur + } + oldhead := h.head + h.commonHook.addHandler(eh, internal) + if oldhead == nil && h.head != nil { + h.p.proc.AddBreakpoint(h.pc) + h.p.breakpointHooks[h.pc] = h + } +} + +func (h *breakpointHook) RemoveHandler(eh EventHandler) { + oldhead := h.head + h.commonHook.RemoveHandler(eh) + if oldhead != nil && h.head == nil { + h.p.proc.RemoveBreakpoint(h.pc) + h.p.breakpointHooks[h.pc] = nil, false + } +} + +func (h *breakpointHook) String() string { + // TODO(austin) Include process name? + // TODO(austin) Use line:pc or at least sym+%#x + return fmt.Sprintf("breakpoint at %#x", h.pc) +} + +func (b *Breakpoint) PC() proc.Word { return b.pc } + +func (b *Breakpoint) String() string { + // TODO(austin) Include process name and goroutine + // TODO(austin) Use line:pc or at least sym+%#x + return fmt.Sprintf("breakpoint at %#x", b.pc) +} + +/* + * Goroutine create/exit + */ + +type goroutineCreateHook struct { + commonHook +} + +func (h *goroutineCreateHook) String() string { return "goroutine create" } + +// A GoroutineCreate event occurs when a process creates a new +// goroutine. When this event is handled, the current goroutine will +// be the newly created goroutine. +type GoroutineCreate struct { + commonEvent + parent *Goroutine +} + +// Parent returns the goroutine that created this goroutine. May be +// nil if this event is the creation of the first goroutine. +func (e *GoroutineCreate) Parent() *Goroutine { return e.parent } + +func (e *GoroutineCreate) String() string { + // TODO(austin) Include process name + if e.parent == nil { + return fmt.Sprintf("%v created", e.t) + } + return fmt.Sprintf("%v created by %v", e.t, e.parent) +} + +type goroutineExitHook struct { + commonHook +} + +func (h *goroutineExitHook) String() string { return "goroutine exit" } + +// A GoroutineExit event occurs when a Go goroutine exits. +type GoroutineExit struct { + commonEvent +} + +func (e *GoroutineExit) String() string { + // TODO(austin) Include process name + //return fmt.Sprintf("%v exited", e.t); + // For debugging purposes + return fmt.Sprintf("goroutine %#x exited", e.t.g.addr().base) +} diff --git a/libgo/go/exp/ogle/frame.go b/libgo/go/exp/ogle/frame.go new file mode 100644 index 000000000..1538362ba --- /dev/null +++ b/libgo/go/exp/ogle/frame.go @@ -0,0 +1,212 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ogle + +import ( + "debug/gosym" + "debug/proc" + "fmt" + "os" +) + +// A Frame represents a single frame on a remote call stack. +type Frame struct { + // pc is the PC of the next instruction that will execute in + // this frame. For lower frames, this is the instruction + // following the CALL instruction. + pc, sp, fp proc.Word + // The runtime.Stktop of the active stack segment + stk remoteStruct + // The function this stack frame is in + fn *gosym.Func + // The path and line of the CALL or current instruction. Note + // that this differs slightly from the meaning of Frame.pc. + path string + line int + // The inner and outer frames of this frame. outer is filled + // in lazily. + inner, outer *Frame +} + +// newFrame returns the top-most Frame of the given g's thread. +func newFrame(g remoteStruct) (*Frame, os.Error) { + var f *Frame + err := try(func(a aborter) { f = aNewFrame(a, g) }) + return f, err +} + +func aNewFrame(a aborter, g remoteStruct) *Frame { + p := g.r.p + var pc, sp proc.Word + + // Is this G alive? + switch g.field(p.f.G.Status).(remoteInt).aGet(a) { + case p.runtime.Gidle, p.runtime.Gmoribund, p.runtime.Gdead: + return nil + } + + // Find the OS thread for this G + + // TODO(austin) Ideally, we could look at the G's state and + // figure out if it's on an OS thread or not. However, this + // is difficult because the state isn't updated atomically + // with scheduling changes. + for _, t := range p.proc.Threads() { + regs, err := t.Regs() + if err != nil { + // TODO(austin) What to do? + continue + } + thisg := p.G(regs) + if thisg == g.addr().base { + // Found this G's OS thread + pc = regs.PC() + sp = regs.SP() + + // If this thread crashed, try to recover it + if pc == 0 { + pc = p.peekUintptr(a, pc) + sp += 8 + } + + break + } + } + + if pc == 0 && sp == 0 { + // G is not mapped to an OS thread. Use the + // scheduler's stored PC and SP. + sched := g.field(p.f.G.Sched).(remoteStruct) + pc = proc.Word(sched.field(p.f.Gobuf.Pc).(remoteUint).aGet(a)) + sp = proc.Word(sched.field(p.f.Gobuf.Sp).(remoteUint).aGet(a)) + } + + // Get Stktop + stk := g.field(p.f.G.Stackbase).(remotePtr).aGet(a).(remoteStruct) + + return prepareFrame(a, pc, sp, stk, nil) +} + +// prepareFrame creates a Frame from the PC and SP within that frame, +// as well as the active stack segment. This function takes care of +// traversing stack breaks and unwinding closures. +func prepareFrame(a aborter, pc, sp proc.Word, stk remoteStruct, inner *Frame) *Frame { + // Based on src/pkg/runtime/amd64/traceback.c:traceback + p := stk.r.p + top := inner == nil + + // Get function + var path string + var line int + var fn *gosym.Func + + for i := 0; i < 100; i++ { + // Traverse segmented stack breaks + if p.sys.lessstack != nil && pc == proc.Word(p.sys.lessstack.Value) { + // Get stk->gobuf.pc + pc = proc.Word(stk.field(p.f.Stktop.Gobuf).(remoteStruct).field(p.f.Gobuf.Pc).(remoteUint).aGet(a)) + // Get stk->gobuf.sp + sp = proc.Word(stk.field(p.f.Stktop.Gobuf).(remoteStruct).field(p.f.Gobuf.Sp).(remoteUint).aGet(a)) + // Get stk->stackbase + stk = stk.field(p.f.Stktop.Stackbase).(remotePtr).aGet(a).(remoteStruct) + continue + } + + // Get the PC of the call instruction + callpc := pc + if !top && (p.sys.goexit == nil || pc != proc.Word(p.sys.goexit.Value)) { + callpc-- + } + + // Look up function + path, line, fn = p.syms.PCToLine(uint64(callpc)) + if fn != nil { + break + } + + // Closure? + var buf = make([]byte, p.ClosureSize()) + if _, err := p.Peek(pc, buf); err != nil { + break + } + spdelta, ok := p.ParseClosure(buf) + if ok { + sp += proc.Word(spdelta) + pc = p.peekUintptr(a, sp-proc.Word(p.PtrSize())) + } + } + if fn == nil { + return nil + } + + // Compute frame pointer + var fp proc.Word + if fn.FrameSize < p.PtrSize() { + fp = sp + proc.Word(p.PtrSize()) + } else { + fp = sp + proc.Word(fn.FrameSize) + } + // TODO(austin) To really figure out if we're in the prologue, + // we need to disassemble the function and look for the call + // to morestack. For now, just special case the entry point. + // + // TODO(austin) What if we're in the call to morestack in the + // prologue? Then top == false. + if top && pc == proc.Word(fn.Entry) { + // We're in the function prologue, before SP + // has been adjusted for the frame. + fp -= proc.Word(fn.FrameSize - p.PtrSize()) + } + + return &Frame{pc, sp, fp, stk, fn, path, line, inner, nil} +} + +// Outer returns the Frame that called this Frame, or nil if this is +// the outermost frame. +func (f *Frame) Outer() (*Frame, os.Error) { + var fr *Frame + err := try(func(a aborter) { fr = f.aOuter(a) }) + return fr, err +} + +func (f *Frame) aOuter(a aborter) *Frame { + // Is there a cached outer frame + if f.outer != nil { + return f.outer + } + + p := f.stk.r.p + + sp := f.fp + if f.fn == p.sys.newproc && f.fn == p.sys.deferproc { + // TODO(rsc) The compiler inserts two push/pop's + // around calls to go and defer. Russ says this + // should get fixed in the compiler, but we account + // for it for now. + sp += proc.Word(2 * p.PtrSize()) + } + + pc := p.peekUintptr(a, f.fp-proc.Word(p.PtrSize())) + if pc < 0x1000 { + return nil + } + + // TODO(austin) Register this frame for shoot-down. + + f.outer = prepareFrame(a, pc, sp, f.stk, f) + return f.outer +} + +// Inner returns the Frame called by this Frame, or nil if this is the +// innermost frame. +func (f *Frame) Inner() *Frame { return f.inner } + +func (f *Frame) String() string { + res := f.fn.Name + if f.pc > proc.Word(f.fn.Value) { + res += fmt.Sprintf("+%#x", f.pc-proc.Word(f.fn.Entry)) + } + return res + fmt.Sprintf(" %s:%d", f.path, f.line) +} diff --git a/libgo/go/exp/ogle/goroutine.go b/libgo/go/exp/ogle/goroutine.go new file mode 100644 index 000000000..5104ec6d4 --- /dev/null +++ b/libgo/go/exp/ogle/goroutine.go @@ -0,0 +1,117 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ogle + +import ( + "debug/proc" + "exp/eval" + "fmt" + "os" +) + +// A Goroutine represents a goroutine in a remote process. +type Goroutine struct { + g remoteStruct + frame *Frame + dead bool +} + +func (t *Goroutine) String() string { + if t.dead { + return "" + } + // TODO(austin) Give threads friendly ID's, possibly including + // the name of the entry function. + return fmt.Sprintf("thread %#x", t.g.addr().base) +} + +// isG0 returns true if this thread if the internal idle thread +func (t *Goroutine) isG0() bool { return t.g.addr().base == t.g.r.p.sys.g0.addr().base } + +func (t *Goroutine) resetFrame() (err os.Error) { + // TODO(austin) Reuse any live part of the current frame stack + // so existing references to Frame's keep working. + t.frame, err = newFrame(t.g) + return +} + +// Out selects the caller frame of the current frame. +func (t *Goroutine) Out() os.Error { + f, err := t.frame.Outer() + if f != nil { + t.frame = f + } + return err +} + +// In selects the frame called by the current frame. +func (t *Goroutine) In() os.Error { + f := t.frame.Inner() + if f != nil { + t.frame = f + } + return nil +} + +func readylockedBP(ev Event) (EventAction, os.Error) { + b := ev.(*Breakpoint) + p := b.Process() + + // The new g is the only argument to this function, so the + // stack will have the return address, then the G*. + regs, err := b.osThread.Regs() + if err != nil { + return EAStop, err + } + sp := regs.SP() + addr := sp + proc.Word(p.PtrSize()) + arg := remotePtr{remote{addr, p}, p.runtime.G} + var gp eval.Value + err = try(func(a aborter) { gp = arg.aGet(a) }) + if err != nil { + return EAStop, err + } + if gp == nil { + return EAStop, UnknownGoroutine{b.osThread, 0} + } + gs := gp.(remoteStruct) + g := &Goroutine{gs, nil, false} + p.goroutines[gs.addr().base] = g + + // Enqueue goroutine creation event + parent := b.Goroutine() + if parent.isG0() { + parent = nil + } + p.postEvent(&GoroutineCreate{commonEvent{p, g}, parent}) + + // If we don't have any thread selected, select this one + if p.curGoroutine == nil { + p.curGoroutine = g + } + + return EADefault, nil +} + +func goexitBP(ev Event) (EventAction, os.Error) { + b := ev.(*Breakpoint) + p := b.Process() + + g := b.Goroutine() + g.dead = true + + addr := g.g.addr().base + p.goroutines[addr] = nil, false + + // Enqueue thread exit event + p.postEvent(&GoroutineExit{commonEvent{p, g}}) + + // If we just exited our selected goroutine, selected another + if p.curGoroutine == g { + p.selectSomeGoroutine() + } + + return EADefault, nil +} diff --git a/libgo/go/exp/ogle/main.go b/libgo/go/exp/ogle/main.go new file mode 100644 index 000000000..1999eccca --- /dev/null +++ b/libgo/go/exp/ogle/main.go @@ -0,0 +1,9 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import "exp/ogle" + +func main() { ogle.Main() } diff --git a/libgo/go/exp/ogle/process.go b/libgo/go/exp/ogle/process.go new file mode 100644 index 000000000..58e830aa6 --- /dev/null +++ b/libgo/go/exp/ogle/process.go @@ -0,0 +1,521 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ogle + +import ( + "debug/elf" + "debug/gosym" + "debug/proc" + "exp/eval" + "fmt" + "log" + "os" + "reflect" +) + +// A FormatError indicates a failure to process information in or +// about a remote process, such as unexpected or missing information +// in the object file or runtime structures. +type FormatError string + +func (e FormatError) String() string { return string(e) } + +// An UnknownArchitecture occurs when trying to load an object file +// that indicates an architecture not supported by the debugger. +type UnknownArchitecture elf.Machine + +func (e UnknownArchitecture) String() string { + return "unknown architecture: " + elf.Machine(e).String() +} + +// A ProcessNotStopped error occurs when attempting to read or write +// memory or registers of a process that is not stopped. +type ProcessNotStopped struct{} + +func (e ProcessNotStopped) String() string { return "process not stopped" } + +// An UnknownGoroutine error is an internal error representing an +// unrecognized G structure pointer. +type UnknownGoroutine struct { + OSThread proc.Thread + Goroutine proc.Word +} + +func (e UnknownGoroutine) String() string { + return fmt.Sprintf("internal error: unknown goroutine (G %#x)", e.Goroutine) +} + +// A NoCurrentGoroutine error occurs when no goroutine is currently +// selected in a process (or when there are no goroutines in a +// process). +type NoCurrentGoroutine struct{} + +func (e NoCurrentGoroutine) String() string { return "no current goroutine" } + +// A Process represents a remote attached process. +type Process struct { + Arch + proc proc.Process + + // The symbol table of this process + syms *gosym.Table + + // A possibly-stopped OS thread, or nil + threadCache proc.Thread + + // Types parsed from the remote process + types map[proc.Word]*remoteType + + // Types and values from the remote runtime package + runtime runtimeValues + + // Runtime field indexes + f runtimeIndexes + + // Globals from the sys package (or from no package) + sys struct { + lessstack, goexit, newproc, deferproc, newprocreadylocked *gosym.Func + allg remotePtr + g0 remoteStruct + } + + // Event queue + posted []Event + pending []Event + event Event + + // Event hooks + breakpointHooks map[proc.Word]*breakpointHook + goroutineCreateHook *goroutineCreateHook + goroutineExitHook *goroutineExitHook + + // Current goroutine, or nil if there are no goroutines + curGoroutine *Goroutine + + // Goroutines by the address of their G structure + goroutines map[proc.Word]*Goroutine +} + +/* + * Process creation + */ + +// NewProcess constructs a new remote process around a traced +// process, an architecture, and a symbol table. +func NewProcess(tproc proc.Process, arch Arch, syms *gosym.Table) (*Process, os.Error) { + p := &Process{ + Arch: arch, + proc: tproc, + syms: syms, + types: make(map[proc.Word]*remoteType), + breakpointHooks: make(map[proc.Word]*breakpointHook), + goroutineCreateHook: new(goroutineCreateHook), + goroutineExitHook: new(goroutineExitHook), + goroutines: make(map[proc.Word]*Goroutine), + } + + // Fill in remote runtime + p.bootstrap() + + switch { + case p.sys.allg.addr().base == 0: + return nil, FormatError("failed to find runtime symbol 'allg'") + case p.sys.g0.addr().base == 0: + return nil, FormatError("failed to find runtime symbol 'g0'") + case p.sys.newprocreadylocked == nil: + return nil, FormatError("failed to find runtime symbol 'newprocreadylocked'") + case p.sys.goexit == nil: + return nil, FormatError("failed to find runtime symbol 'sys.goexit'") + } + + // Get current goroutines + p.goroutines[p.sys.g0.addr().base] = &Goroutine{p.sys.g0, nil, false} + err := try(func(a aborter) { + g := p.sys.allg.aGet(a) + for g != nil { + gs := g.(remoteStruct) + fmt.Printf("*** Found goroutine at %#x\n", gs.addr().base) + p.goroutines[gs.addr().base] = &Goroutine{gs, nil, false} + g = gs.field(p.f.G.Alllink).(remotePtr).aGet(a) + } + }) + if err != nil { + return nil, err + } + + // Create internal breakpoints to catch new and exited goroutines + p.OnBreakpoint(proc.Word(p.sys.newprocreadylocked.Entry)).(*breakpointHook).addHandler(readylockedBP, true) + p.OnBreakpoint(proc.Word(p.sys.goexit.Entry)).(*breakpointHook).addHandler(goexitBP, true) + + // Select current frames + for _, g := range p.goroutines { + g.resetFrame() + } + + p.selectSomeGoroutine() + + return p, nil +} + +func elfGoSyms(f *elf.File) (*gosym.Table, os.Error) { + text := f.Section(".text") + symtab := f.Section(".gosymtab") + pclntab := f.Section(".gopclntab") + if text == nil || symtab == nil || pclntab == nil { + return nil, nil + } + + symdat, err := symtab.Data() + if err != nil { + return nil, err + } + pclndat, err := pclntab.Data() + if err != nil { + return nil, err + } + + pcln := gosym.NewLineTable(pclndat, text.Addr) + tab, err := gosym.NewTable(symdat, pcln) + if err != nil { + return nil, err + } + + return tab, nil +} + +// NewProcessElf constructs a new remote process around a traced +// process and the process' ELF object. +func NewProcessElf(tproc proc.Process, f *elf.File) (*Process, os.Error) { + syms, err := elfGoSyms(f) + if err != nil { + return nil, err + } + if syms == nil { + return nil, FormatError("Failed to find symbol table") + } + var arch Arch + switch f.Machine { + case elf.EM_X86_64: + arch = Amd64 + default: + return nil, UnknownArchitecture(f.Machine) + } + return NewProcess(tproc, arch, syms) +} + +// bootstrap constructs the runtime structure of a remote process. +func (p *Process) bootstrap() { + // Manually construct runtime types + p.runtime.String = newManualType(eval.TypeOfNative(rt1String{}), p.Arch) + p.runtime.Slice = newManualType(eval.TypeOfNative(rt1Slice{}), p.Arch) + p.runtime.Eface = newManualType(eval.TypeOfNative(rt1Eface{}), p.Arch) + + p.runtime.Type = newManualType(eval.TypeOfNative(rt1Type{}), p.Arch) + p.runtime.CommonType = newManualType(eval.TypeOfNative(rt1CommonType{}), p.Arch) + p.runtime.UncommonType = newManualType(eval.TypeOfNative(rt1UncommonType{}), p.Arch) + p.runtime.StructField = newManualType(eval.TypeOfNative(rt1StructField{}), p.Arch) + p.runtime.StructType = newManualType(eval.TypeOfNative(rt1StructType{}), p.Arch) + p.runtime.PtrType = newManualType(eval.TypeOfNative(rt1PtrType{}), p.Arch) + p.runtime.ArrayType = newManualType(eval.TypeOfNative(rt1ArrayType{}), p.Arch) + p.runtime.SliceType = newManualType(eval.TypeOfNative(rt1SliceType{}), p.Arch) + + p.runtime.Stktop = newManualType(eval.TypeOfNative(rt1Stktop{}), p.Arch) + p.runtime.Gobuf = newManualType(eval.TypeOfNative(rt1Gobuf{}), p.Arch) + p.runtime.G = newManualType(eval.TypeOfNative(rt1G{}), p.Arch) + + // Get addresses of type.*runtime.XType for discrimination. + rtv := reflect.Indirect(reflect.NewValue(&p.runtime)).(*reflect.StructValue) + rtvt := rtv.Type().(*reflect.StructType) + for i := 0; i < rtv.NumField(); i++ { + n := rtvt.Field(i).Name + if n[0] != 'P' || n[1] < 'A' || n[1] > 'Z' { + continue + } + sym := p.syms.LookupSym("type.*runtime." + n[1:]) + if sym == nil { + continue + } + rtv.Field(i).(*reflect.UintValue).Set(sym.Value) + } + + // Get runtime field indexes + fillRuntimeIndexes(&p.runtime, &p.f) + + // Fill G status + p.runtime.runtimeGStatus = rt1GStatus + + // Get globals + p.sys.lessstack = p.syms.LookupFunc("sys.lessstack") + p.sys.goexit = p.syms.LookupFunc("goexit") + p.sys.newproc = p.syms.LookupFunc("sys.newproc") + p.sys.deferproc = p.syms.LookupFunc("sys.deferproc") + p.sys.newprocreadylocked = p.syms.LookupFunc("newprocreadylocked") + if allg := p.syms.LookupSym("allg"); allg != nil { + p.sys.allg = remotePtr{remote{proc.Word(allg.Value), p}, p.runtime.G} + } + if g0 := p.syms.LookupSym("g0"); g0 != nil { + p.sys.g0 = p.runtime.G.mk(remote{proc.Word(g0.Value), p}).(remoteStruct) + } +} + +func (p *Process) selectSomeGoroutine() { + // Once we have friendly goroutine ID's, there might be a more + // reasonable behavior for this. + p.curGoroutine = nil + for _, g := range p.goroutines { + if !g.isG0() && g.frame != nil { + p.curGoroutine = g + return + } + } +} + +/* + * Process memory + */ + +func (p *Process) someStoppedOSThread() proc.Thread { + if p.threadCache != nil { + if _, err := p.threadCache.Stopped(); err == nil { + return p.threadCache + } + } + + for _, t := range p.proc.Threads() { + if _, err := t.Stopped(); err == nil { + p.threadCache = t + return t + } + } + return nil +} + +func (p *Process) Peek(addr proc.Word, out []byte) (int, os.Error) { + thr := p.someStoppedOSThread() + if thr == nil { + return 0, ProcessNotStopped{} + } + return thr.Peek(addr, out) +} + +func (p *Process) Poke(addr proc.Word, b []byte) (int, os.Error) { + thr := p.someStoppedOSThread() + if thr == nil { + return 0, ProcessNotStopped{} + } + return thr.Poke(addr, b) +} + +func (p *Process) peekUintptr(a aborter, addr proc.Word) proc.Word { + return proc.Word(mkUintptr(remote{addr, p}).(remoteUint).aGet(a)) +} + +/* + * Events + */ + +// OnBreakpoint returns the hook that is run when the program reaches +// the given program counter. +func (p *Process) OnBreakpoint(pc proc.Word) EventHook { + if bp, ok := p.breakpointHooks[pc]; ok { + return bp + } + // The breakpoint will register itself when a handler is added + return &breakpointHook{commonHook{nil, 0}, p, pc} +} + +// OnGoroutineCreate returns the hook that is run when a goroutine is created. +func (p *Process) OnGoroutineCreate() EventHook { + return p.goroutineCreateHook +} + +// OnGoroutineExit returns the hook that is run when a goroutine exits. +func (p *Process) OnGoroutineExit() EventHook { return p.goroutineExitHook } + +// osThreadToGoroutine looks up the goroutine running on an OS thread. +func (p *Process) osThreadToGoroutine(t proc.Thread) (*Goroutine, os.Error) { + regs, err := t.Regs() + if err != nil { + return nil, err + } + g := p.G(regs) + gt, ok := p.goroutines[g] + if !ok { + return nil, UnknownGoroutine{t, g} + } + return gt, nil +} + +// causesToEvents translates the stop causes of the underlying process +// into an event queue. +func (p *Process) causesToEvents() ([]Event, os.Error) { + // Count causes we're interested in + nev := 0 + for _, t := range p.proc.Threads() { + if c, err := t.Stopped(); err == nil { + switch c := c.(type) { + case proc.Breakpoint: + nev++ + case proc.Signal: + // TODO(austin) + //nev++; + } + } + } + + // Translate causes to events + events := make([]Event, nev) + i := 0 + for _, t := range p.proc.Threads() { + if c, err := t.Stopped(); err == nil { + switch c := c.(type) { + case proc.Breakpoint: + gt, err := p.osThreadToGoroutine(t) + if err != nil { + return nil, err + } + events[i] = &Breakpoint{commonEvent{p, gt}, t, proc.Word(c)} + i++ + case proc.Signal: + // TODO(austin) + } + } + } + + return events, nil +} + +// postEvent appends an event to the posted queue. These events will +// be processed before any currently pending events. +func (p *Process) postEvent(ev Event) { + p.posted = append(p.posted, ev) +} + +// processEvents processes events in the event queue until no events +// remain, a handler returns EAStop, or a handler returns an error. +// It returns either EAStop or EAContinue and possibly an error. +func (p *Process) processEvents() (EventAction, os.Error) { + var ev Event + for len(p.posted) > 0 { + ev, p.posted = p.posted[0], p.posted[1:] + action, err := p.processEvent(ev) + if action == EAStop { + return action, err + } + } + + for len(p.pending) > 0 { + ev, p.pending = p.pending[0], p.pending[1:] + action, err := p.processEvent(ev) + if action == EAStop { + return action, err + } + } + + return EAContinue, nil +} + +// processEvent processes a single event, without manipulating the +// event queues. It returns either EAStop or EAContinue and possibly +// an error. +func (p *Process) processEvent(ev Event) (EventAction, os.Error) { + p.event = ev + + var action EventAction + var err os.Error + switch ev := p.event.(type) { + case *Breakpoint: + hook, ok := p.breakpointHooks[ev.pc] + if !ok { + break + } + p.curGoroutine = ev.Goroutine() + action, err = hook.handle(ev) + + case *GoroutineCreate: + p.curGoroutine = ev.Goroutine() + action, err = p.goroutineCreateHook.handle(ev) + + case *GoroutineExit: + action, err = p.goroutineExitHook.handle(ev) + + default: + log.Panicf("Unknown event type %T in queue", p.event) + } + + if err != nil { + return EAStop, err + } else if action == EAStop { + return EAStop, nil + } + return EAContinue, nil +} + +// Event returns the last event that caused the process to stop. This +// may return nil if the process has never been stopped by an event. +// +// TODO(austin) Return nil if the user calls p.Stop()? +func (p *Process) Event() Event { return p.event } + +/* + * Process control + */ + +// TODO(austin) Cont, WaitStop, and Stop. Need to figure out how +// event handling works with these. Originally I did it only in +// WaitStop, but if you Cont and there are pending events, then you +// have to not actually continue and wait until a WaitStop to process +// them, even if the event handlers will tell you to continue. We +// could handle them in both Cont and WaitStop to avoid this problem, +// but it's still weird if an event happens after the Cont and before +// the WaitStop that the handlers say to continue from. Or we could +// handle them on a separate thread. Then obviously you get weird +// asynchronous things, like prints while the user it typing a command, +// but that's not necessarily a bad thing. + +// ContWait resumes process execution and waits for an event to occur +// that stops the process. +func (p *Process) ContWait() os.Error { + for { + a, err := p.processEvents() + if err != nil { + return err + } else if a == EAStop { + break + } + err = p.proc.Continue() + if err != nil { + return err + } + err = p.proc.WaitStop() + if err != nil { + return err + } + for _, g := range p.goroutines { + g.resetFrame() + } + p.pending, err = p.causesToEvents() + if err != nil { + return err + } + } + return nil +} + +// Out selects the caller frame of the current frame. +func (p *Process) Out() os.Error { + if p.curGoroutine == nil { + return NoCurrentGoroutine{} + } + return p.curGoroutine.Out() +} + +// In selects the frame called by the current frame. +func (p *Process) In() os.Error { + if p.curGoroutine == nil { + return NoCurrentGoroutine{} + } + return p.curGoroutine.In() +} diff --git a/libgo/go/exp/ogle/rruntime.go b/libgo/go/exp/ogle/rruntime.go new file mode 100644 index 000000000..33f1935b8 --- /dev/null +++ b/libgo/go/exp/ogle/rruntime.go @@ -0,0 +1,271 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ogle + +import ( + "debug/proc" + "exp/eval" + "reflect" +) + +// This file contains remote runtime definitions. Using reflection, +// we convert all of these to interpreter types and layout their +// remote representations using the architecture rules. +// +// We could get most of these definitions from our own runtime +// package; however, some of them differ in convenient ways, some of +// them are not defined or exported by the runtime, and having our own +// definitions makes it easy to support multiple remote runtime +// versions. This may turn out to be overkill. +// +// All of these structures are prefixed with rt1 to indicate the +// runtime version and to mark them as types used only as templates +// for remote types. + +/* + * Runtime data headers + * + * See $GOROOT/src/pkg/runtime/runtime.h + */ + +type rt1String struct { + str uintptr + len int +} + +type rt1Slice struct { + array uintptr + len int + cap int +} + +type rt1Eface struct { + typ uintptr + ptr uintptr +} + +/* + * Runtime type structures + * + * See $GOROOT/src/pkg/runtime/type.h and $GOROOT/src/pkg/runtime/type.go + */ + +type rt1UncommonType struct { + name *string + pkgPath *string + //methods []method; +} + +type rt1CommonType struct { + size uintptr + hash uint32 + alg, align, fieldAlign uint8 + string *string + uncommonType *rt1UncommonType +} + +type rt1Type struct { + // While Type is technically an Eface, treating the + // discriminator as an opaque pointer and taking advantage of + // the commonType prologue on all Type's makes type parsing + // much simpler. + typ uintptr + ptr *rt1CommonType +} + +type rt1StructField struct { + name *string + pkgPath *string + typ *rt1Type + tag *string + offset uintptr +} + +type rt1StructType struct { + rt1CommonType + fields []rt1StructField +} + +type rt1PtrType struct { + rt1CommonType + elem *rt1Type +} + +type rt1SliceType struct { + rt1CommonType + elem *rt1Type +} + +type rt1ArrayType struct { + rt1CommonType + elem *rt1Type + len uintptr +} + +/* + * Runtime scheduler structures + * + * See $GOROOT/src/pkg/runtime/runtime.h + */ + +// Fields beginning with _ are only for padding + +type rt1Stktop struct { + stackguard uintptr + stackbase *rt1Stktop + gobuf rt1Gobuf + _args uint32 + _fp uintptr +} + +type rt1Gobuf struct { + sp uintptr + pc uintptr + g *rt1G + r0 uintptr +} + +type rt1G struct { + _stackguard uintptr + stackbase *rt1Stktop + _defer uintptr + sched rt1Gobuf + _stack0 uintptr + _entry uintptr + alllink *rt1G + _param uintptr + status int16 + // Incomplete +} + +var rt1GStatus = runtimeGStatus{ + Gidle: 0, + Grunnable: 1, + Grunning: 2, + Gsyscall: 3, + Gwaiting: 4, + Gmoribund: 5, + Gdead: 6, +} + +// runtimeIndexes stores the indexes of fields in the runtime +// structures. It is filled in using reflection, so the name of the +// fields must match the names of the remoteType's in runtimeValues +// exactly and the names of the index fields must be the capitalized +// version of the names of the fields in the runtime structures above. +type runtimeIndexes struct { + String struct { + Str, Len int + } + Slice struct { + Array, Len, Cap int + } + Eface struct { + Typ, Ptr int + } + + UncommonType struct { + Name, PkgPath int + } + CommonType struct { + Size, Hash, Alg, Align, FieldAlign, String, UncommonType int + } + Type struct { + Typ, Ptr int + } + StructField struct { + Name, PkgPath, Typ, Tag, Offset int + } + StructType struct { + Fields int + } + PtrType struct { + Elem int + } + SliceType struct { + Elem int + } + ArrayType struct { + Elem, Len int + } + + Stktop struct { + Stackguard, Stackbase, Gobuf int + } + Gobuf struct { + Sp, Pc, G int + } + G struct { + Stackbase, Sched, Status, Alllink int + } +} + +// Values of G status codes +type runtimeGStatus struct { + Gidle, Grunnable, Grunning, Gsyscall, Gwaiting, Gmoribund, Gdead int64 +} + +// runtimeValues stores the types and values that correspond to those +// in the remote runtime package. +type runtimeValues struct { + // Runtime data headers + String, Slice, Eface *remoteType + // Runtime type structures + Type, CommonType, UncommonType, StructField, StructType, PtrType, + ArrayType, SliceType *remoteType + // Runtime scheduler structures + Stktop, Gobuf, G *remoteType + // Addresses of *runtime.XType types. These are the + // discriminators on the runtime.Type interface. We use local + // reflection to fill these in from the remote symbol table, + // so the names must match the runtime names. + PBoolType, + PUint8Type, PUint16Type, PUint32Type, PUint64Type, PUintType, PUintptrType, + PInt8Type, PInt16Type, PInt32Type, PInt64Type, PIntType, + PFloat32Type, PFloat64Type, PFloatType, + PArrayType, PStringType, PStructType, PPtrType, PFuncType, + PInterfaceType, PSliceType, PMapType, PChanType, + PDotDotDotType, PUnsafePointerType proc.Word + // G status values + runtimeGStatus +} + +// fillRuntimeIndexes fills a runtimeIndexes structure will the field +// indexes gathered from the remoteTypes recorded in a runtimeValues +// structure. +func fillRuntimeIndexes(runtime *runtimeValues, out *runtimeIndexes) { + outv := reflect.Indirect(reflect.NewValue(out)).(*reflect.StructValue) + outt := outv.Type().(*reflect.StructType) + runtimev := reflect.Indirect(reflect.NewValue(runtime)).(*reflect.StructValue) + + // out contains fields corresponding to each runtime type + for i := 0; i < outt.NumField(); i++ { + // Find the interpreter type for this runtime type + name := outt.Field(i).Name + et := runtimev.FieldByName(name).Interface().(*remoteType).Type.(*eval.StructType) + + // Get the field indexes of the interpreter struct type + indexes := make(map[string]int, len(et.Elems)) + for j, f := range et.Elems { + if f.Anonymous { + continue + } + name := f.Name + if name[0] >= 'a' && name[0] <= 'z' { + name = string(name[0]+'A'-'a') + name[1:] + } + indexes[name] = j + } + + // Fill this field of out + outStructv := outv.Field(i).(*reflect.StructValue) + outStructt := outStructv.Type().(*reflect.StructType) + for j := 0; j < outStructt.NumField(); j++ { + f := outStructv.Field(j).(*reflect.IntValue) + name := outStructt.Field(j).Name + f.Set(int64(indexes[name])) + } + } +} diff --git a/libgo/go/exp/ogle/rtype.go b/libgo/go/exp/ogle/rtype.go new file mode 100644 index 000000000..b3c35575a --- /dev/null +++ b/libgo/go/exp/ogle/rtype.go @@ -0,0 +1,288 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ogle + +import ( + "debug/proc" + "exp/eval" + "fmt" + "log" +) + +const debugParseRemoteType = false + +// A remoteType is the local representation of a type in a remote process. +type remoteType struct { + eval.Type + // The size of values of this type in bytes. + size int + // The field alignment of this type. Only used for + // manually-constructed types. + fieldAlign int + // The maker function to turn a remote address of a value of + // this type into an interpreter Value. + mk maker +} + +var manualTypes = make(map[Arch]map[eval.Type]*remoteType) + +// newManualType constructs a remote type from an interpreter Type +// using the size and alignment properties of the given architecture. +// Most types are parsed directly out of the remote process, but to do +// so we need to layout the structures that describe those types ourselves. +func newManualType(t eval.Type, arch Arch) *remoteType { + if nt, ok := t.(*eval.NamedType); ok { + t = nt.Def + } + + // Get the type map for this architecture + typeMap := manualTypes[arch] + if typeMap == nil { + typeMap = make(map[eval.Type]*remoteType) + manualTypes[arch] = typeMap + + // Construct basic types for this architecture + basicType := func(t eval.Type, mk maker, size int, fieldAlign int) { + t = t.(*eval.NamedType).Def + if fieldAlign == 0 { + fieldAlign = size + } + typeMap[t] = &remoteType{t, size, fieldAlign, mk} + } + basicType(eval.Uint8Type, mkUint8, 1, 0) + basicType(eval.Uint32Type, mkUint32, 4, 0) + basicType(eval.UintptrType, mkUintptr, arch.PtrSize(), 0) + basicType(eval.Int16Type, mkInt16, 2, 0) + basicType(eval.Int32Type, mkInt32, 4, 0) + basicType(eval.IntType, mkInt, arch.IntSize(), 0) + basicType(eval.StringType, mkString, arch.PtrSize()+arch.IntSize(), arch.PtrSize()) + } + + if rt, ok := typeMap[t]; ok { + return rt + } + + var rt *remoteType + switch t := t.(type) { + case *eval.PtrType: + var elem *remoteType + mk := func(r remote) eval.Value { return remotePtr{r, elem} } + rt = &remoteType{t, arch.PtrSize(), arch.PtrSize(), mk} + // Construct the element type after registering the + // type to break cycles. + typeMap[eval.Type(t)] = rt + elem = newManualType(t.Elem, arch) + + case *eval.ArrayType: + elem := newManualType(t.Elem, arch) + mk := func(r remote) eval.Value { return remoteArray{r, t.Len, elem} } + rt = &remoteType{t, elem.size * int(t.Len), elem.fieldAlign, mk} + + case *eval.SliceType: + elem := newManualType(t.Elem, arch) + mk := func(r remote) eval.Value { return remoteSlice{r, elem} } + rt = &remoteType{t, arch.PtrSize() + 2*arch.IntSize(), arch.PtrSize(), mk} + + case *eval.StructType: + layout := make([]remoteStructField, len(t.Elems)) + offset := 0 + fieldAlign := 0 + for i, f := range t.Elems { + elem := newManualType(f.Type, arch) + if fieldAlign == 0 { + fieldAlign = elem.fieldAlign + } + offset = arch.Align(offset, elem.fieldAlign) + layout[i].offset = offset + layout[i].fieldType = elem + offset += elem.size + } + mk := func(r remote) eval.Value { return remoteStruct{r, layout} } + rt = &remoteType{t, offset, fieldAlign, mk} + + default: + log.Panicf("cannot manually construct type %T", t) + } + + typeMap[t] = rt + return rt +} + +var prtIndent = "" + +// parseRemoteType parses a Type structure in a remote process to +// construct the corresponding interpreter type and remote type. +func parseRemoteType(a aborter, rs remoteStruct) *remoteType { + addr := rs.addr().base + p := rs.addr().p + + // We deal with circular types by discovering cycles at + // NamedTypes. If a type cycles back to something other than + // a named type, we're guaranteed that there will be a named + // type somewhere in that cycle. Thus, we continue down, + // re-parsing types until we reach the named type in the + // cycle. In order to still create one remoteType per remote + // type, we insert an empty remoteType in the type map the + // first time we encounter the type and re-use that structure + // the second time we encounter it. + + rt, ok := p.types[addr] + if ok && rt.Type != nil { + return rt + } else if !ok { + rt = &remoteType{} + p.types[addr] = rt + } + + if debugParseRemoteType { + sym := p.syms.SymByAddr(uint64(addr)) + name := "" + if sym != nil { + name = sym.Name + } + log.Printf("%sParsing type at %#x (%s)", prtIndent, addr, name) + prtIndent += " " + defer func() { prtIndent = prtIndent[0 : len(prtIndent)-1] }() + } + + // Get Type header + itype := proc.Word(rs.field(p.f.Type.Typ).(remoteUint).aGet(a)) + typ := rs.field(p.f.Type.Ptr).(remotePtr).aGet(a).(remoteStruct) + + // Is this a named type? + var nt *eval.NamedType + uncommon := typ.field(p.f.CommonType.UncommonType).(remotePtr).aGet(a) + if uncommon != nil { + name := uncommon.(remoteStruct).field(p.f.UncommonType.Name).(remotePtr).aGet(a) + if name != nil { + // TODO(austin) Declare type in appropriate remote package + nt = eval.NewNamedType(name.(remoteString).aGet(a)) + rt.Type = nt + } + } + + // Create type + var t eval.Type + var mk maker + switch itype { + case p.runtime.PBoolType: + t = eval.BoolType + mk = mkBool + case p.runtime.PUint8Type: + t = eval.Uint8Type + mk = mkUint8 + case p.runtime.PUint16Type: + t = eval.Uint16Type + mk = mkUint16 + case p.runtime.PUint32Type: + t = eval.Uint32Type + mk = mkUint32 + case p.runtime.PUint64Type: + t = eval.Uint64Type + mk = mkUint64 + case p.runtime.PUintType: + t = eval.UintType + mk = mkUint + case p.runtime.PUintptrType: + t = eval.UintptrType + mk = mkUintptr + case p.runtime.PInt8Type: + t = eval.Int8Type + mk = mkInt8 + case p.runtime.PInt16Type: + t = eval.Int16Type + mk = mkInt16 + case p.runtime.PInt32Type: + t = eval.Int32Type + mk = mkInt32 + case p.runtime.PInt64Type: + t = eval.Int64Type + mk = mkInt64 + case p.runtime.PIntType: + t = eval.IntType + mk = mkInt + case p.runtime.PFloat32Type: + t = eval.Float32Type + mk = mkFloat32 + case p.runtime.PFloat64Type: + t = eval.Float64Type + mk = mkFloat64 + case p.runtime.PStringType: + t = eval.StringType + mk = mkString + + case p.runtime.PArrayType: + // Cast to an ArrayType + typ := p.runtime.ArrayType.mk(typ.addr()).(remoteStruct) + len := int64(typ.field(p.f.ArrayType.Len).(remoteUint).aGet(a)) + elem := parseRemoteType(a, typ.field(p.f.ArrayType.Elem).(remotePtr).aGet(a).(remoteStruct)) + t = eval.NewArrayType(len, elem.Type) + mk = func(r remote) eval.Value { return remoteArray{r, len, elem} } + + case p.runtime.PStructType: + // Cast to a StructType + typ := p.runtime.StructType.mk(typ.addr()).(remoteStruct) + fs := typ.field(p.f.StructType.Fields).(remoteSlice).aGet(a) + + fields := make([]eval.StructField, fs.Len) + layout := make([]remoteStructField, fs.Len) + for i := range fields { + f := fs.Base.(remoteArray).elem(int64(i)).(remoteStruct) + elemrs := f.field(p.f.StructField.Typ).(remotePtr).aGet(a).(remoteStruct) + elem := parseRemoteType(a, elemrs) + fields[i].Type = elem.Type + name := f.field(p.f.StructField.Name).(remotePtr).aGet(a) + if name == nil { + fields[i].Anonymous = true + } else { + fields[i].Name = name.(remoteString).aGet(a) + } + layout[i].offset = int(f.field(p.f.StructField.Offset).(remoteUint).aGet(a)) + layout[i].fieldType = elem + } + + t = eval.NewStructType(fields) + mk = func(r remote) eval.Value { return remoteStruct{r, layout} } + + case p.runtime.PPtrType: + // Cast to a PtrType + typ := p.runtime.PtrType.mk(typ.addr()).(remoteStruct) + elem := parseRemoteType(a, typ.field(p.f.PtrType.Elem).(remotePtr).aGet(a).(remoteStruct)) + t = eval.NewPtrType(elem.Type) + mk = func(r remote) eval.Value { return remotePtr{r, elem} } + + case p.runtime.PSliceType: + // Cast to a SliceType + typ := p.runtime.SliceType.mk(typ.addr()).(remoteStruct) + elem := parseRemoteType(a, typ.field(p.f.SliceType.Elem).(remotePtr).aGet(a).(remoteStruct)) + t = eval.NewSliceType(elem.Type) + mk = func(r remote) eval.Value { return remoteSlice{r, elem} } + + case p.runtime.PMapType, p.runtime.PChanType, p.runtime.PFuncType, p.runtime.PInterfaceType, p.runtime.PUnsafePointerType, p.runtime.PDotDotDotType: + // TODO(austin) + t = eval.UintptrType + mk = mkUintptr + + default: + sym := p.syms.SymByAddr(uint64(itype)) + name := "" + if sym != nil { + name = sym.Name + } + err := fmt.Sprintf("runtime type at %#x has unexpected type %#x (%s)", addr, itype, name) + a.Abort(FormatError(err)) + } + + // Fill in the remote type + if nt != nil { + nt.Complete(t) + } else { + rt.Type = t + } + rt.size = int(typ.field(p.f.CommonType.Size).(remoteUint).aGet(a)) + rt.mk = mk + + return rt +} diff --git a/libgo/go/exp/ogle/rvalue.go b/libgo/go/exp/ogle/rvalue.go new file mode 100644 index 000000000..3d630f936 --- /dev/null +++ b/libgo/go/exp/ogle/rvalue.go @@ -0,0 +1,515 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ogle + +import ( + "debug/proc" + "exp/eval" + "fmt" +) + +// A RemoteMismatchError occurs when an operation that requires two +// identical remote processes is given different process. For +// example, this occurs when trying to set a pointer in one process to +// point to something in another process. +type RemoteMismatchError string + +func (e RemoteMismatchError) String() string { return string(e) } + +// A ReadOnlyError occurs when attempting to set or assign to a +// read-only value. +type ReadOnlyError string + +func (e ReadOnlyError) String() string { return string(e) } + +// A maker is a function that converts a remote address into an +// interpreter Value. +type maker func(remote) eval.Value + +type remoteValue interface { + addr() remote +} + +// remote represents an address in a remote process. +type remote struct { + base proc.Word + p *Process +} + +func (v remote) Get(a aborter, size int) uint64 { + // TODO(austin) This variable might temporarily be in a + // register. We could trace the assembly back from the + // current PC, looking for the beginning of the function or a + // call (both of which guarantee that the variable is in + // memory), or an instruction that loads the variable into a + // register. + // + // TODO(austin) If this is a local variable, it might not be + // live at this PC. In fact, because the compiler reuses + // slots, there might even be a different local variable at + // this location right now. A simple solution to both + // problems is to include the range of PC's over which a local + // variable is live in the symbol table. + // + // TODO(austin) We need to prevent the remote garbage + // collector from collecting objects out from under us. + var arr [8]byte + buf := arr[0:size] + _, err := v.p.Peek(v.base, buf) + if err != nil { + a.Abort(err) + } + return uint64(v.p.ToWord(buf)) +} + +func (v remote) Set(a aborter, size int, x uint64) { + var arr [8]byte + buf := arr[0:size] + v.p.FromWord(proc.Word(x), buf) + _, err := v.p.Poke(v.base, buf) + if err != nil { + a.Abort(err) + } +} + +func (v remote) plus(x proc.Word) remote { return remote{v.base + x, v.p} } + +func tryRVString(f func(a aborter) string) string { + var s string + err := try(func(a aborter) { s = f(a) }) + if err != nil { + return fmt.Sprintf("", err) + } + return s +} + +/* + * Bool + */ + +type remoteBool struct { + r remote +} + +func (v remoteBool) String() string { + return tryRVString(func(a aborter) string { return fmt.Sprintf("%v", v.aGet(a)) }) +} + +func (v remoteBool) Assign(t *eval.Thread, o eval.Value) { + v.Set(t, o.(eval.BoolValue).Get(t)) +} + +func (v remoteBool) Get(t *eval.Thread) bool { return v.aGet(t) } + +func (v remoteBool) aGet(a aborter) bool { return v.r.Get(a, 1) != 0 } + +func (v remoteBool) Set(t *eval.Thread, x bool) { + v.aSet(t, x) +} + +func (v remoteBool) aSet(a aborter, x bool) { + if x { + v.r.Set(a, 1, 1) + } else { + v.r.Set(a, 1, 0) + } +} + +func (v remoteBool) addr() remote { return v.r } + +func mkBool(r remote) eval.Value { return remoteBool{r} } + +/* + * Uint + */ + +type remoteUint struct { + r remote + size int +} + +func (v remoteUint) String() string { + return tryRVString(func(a aborter) string { return fmt.Sprintf("%v", v.aGet(a)) }) +} + +func (v remoteUint) Assign(t *eval.Thread, o eval.Value) { + v.Set(t, o.(eval.UintValue).Get(t)) +} + +func (v remoteUint) Get(t *eval.Thread) uint64 { + return v.aGet(t) +} + +func (v remoteUint) aGet(a aborter) uint64 { return v.r.Get(a, v.size) } + +func (v remoteUint) Set(t *eval.Thread, x uint64) { + v.aSet(t, x) +} + +func (v remoteUint) aSet(a aborter, x uint64) { v.r.Set(a, v.size, x) } + +func (v remoteUint) addr() remote { return v.r } + +func mkUint8(r remote) eval.Value { return remoteUint{r, 1} } + +func mkUint16(r remote) eval.Value { return remoteUint{r, 2} } + +func mkUint32(r remote) eval.Value { return remoteUint{r, 4} } + +func mkUint64(r remote) eval.Value { return remoteUint{r, 8} } + +func mkUint(r remote) eval.Value { return remoteUint{r, r.p.IntSize()} } + +func mkUintptr(r remote) eval.Value { return remoteUint{r, r.p.PtrSize()} } + +/* + * Int + */ + +type remoteInt struct { + r remote + size int +} + +func (v remoteInt) String() string { + return tryRVString(func(a aborter) string { return fmt.Sprintf("%v", v.aGet(a)) }) +} + +func (v remoteInt) Assign(t *eval.Thread, o eval.Value) { + v.Set(t, o.(eval.IntValue).Get(t)) +} + +func (v remoteInt) Get(t *eval.Thread) int64 { return v.aGet(t) } + +func (v remoteInt) aGet(a aborter) int64 { return int64(v.r.Get(a, v.size)) } + +func (v remoteInt) Set(t *eval.Thread, x int64) { + v.aSet(t, x) +} + +func (v remoteInt) aSet(a aborter, x int64) { v.r.Set(a, v.size, uint64(x)) } + +func (v remoteInt) addr() remote { return v.r } + +func mkInt8(r remote) eval.Value { return remoteInt{r, 1} } + +func mkInt16(r remote) eval.Value { return remoteInt{r, 2} } + +func mkInt32(r remote) eval.Value { return remoteInt{r, 4} } + +func mkInt64(r remote) eval.Value { return remoteInt{r, 8} } + +func mkInt(r remote) eval.Value { return remoteInt{r, r.p.IntSize()} } + +/* + * Float + */ + +type remoteFloat struct { + r remote + size int +} + +func (v remoteFloat) String() string { + return tryRVString(func(a aborter) string { return fmt.Sprintf("%v", v.aGet(a)) }) +} + +func (v remoteFloat) Assign(t *eval.Thread, o eval.Value) { + v.Set(t, o.(eval.FloatValue).Get(t)) +} + +func (v remoteFloat) Get(t *eval.Thread) float64 { + return v.aGet(t) +} + +func (v remoteFloat) aGet(a aborter) float64 { + bits := v.r.Get(a, v.size) + switch v.size { + case 4: + return float64(v.r.p.ToFloat32(uint32(bits))) + case 8: + return v.r.p.ToFloat64(bits) + } + panic("Unexpected float size") +} + +func (v remoteFloat) Set(t *eval.Thread, x float64) { + v.aSet(t, x) +} + +func (v remoteFloat) aSet(a aborter, x float64) { + var bits uint64 + switch v.size { + case 4: + bits = uint64(v.r.p.FromFloat32(float32(x))) + case 8: + bits = v.r.p.FromFloat64(x) + default: + panic("Unexpected float size") + } + v.r.Set(a, v.size, bits) +} + +func (v remoteFloat) addr() remote { return v.r } + +func mkFloat32(r remote) eval.Value { return remoteFloat{r, 4} } + +func mkFloat64(r remote) eval.Value { return remoteFloat{r, 8} } + +func mkFloat(r remote) eval.Value { return remoteFloat{r, r.p.FloatSize()} } + +/* + * String + */ + +type remoteString struct { + r remote +} + +func (v remoteString) String() string { + return tryRVString(func(a aborter) string { return v.aGet(a) }) +} + +func (v remoteString) Assign(t *eval.Thread, o eval.Value) { + v.Set(t, o.(eval.StringValue).Get(t)) +} + +func (v remoteString) Get(t *eval.Thread) string { + return v.aGet(t) +} + +func (v remoteString) aGet(a aborter) string { + rs := v.r.p.runtime.String.mk(v.r).(remoteStruct) + str := proc.Word(rs.field(v.r.p.f.String.Str).(remoteUint).aGet(a)) + len := rs.field(v.r.p.f.String.Len).(remoteInt).aGet(a) + + bytes := make([]uint8, len) + _, err := v.r.p.Peek(str, bytes) + if err != nil { + a.Abort(err) + } + return string(bytes) +} + +func (v remoteString) Set(t *eval.Thread, x string) { + v.aSet(t, x) +} + +func (v remoteString) aSet(a aborter, x string) { + // TODO(austin) This isn't generally possible without the + // ability to allocate remote memory. + a.Abort(ReadOnlyError("remote strings cannot be assigned to")) +} + +func mkString(r remote) eval.Value { return remoteString{r} } + +/* + * Array + */ + +type remoteArray struct { + r remote + len int64 + elemType *remoteType +} + +func (v remoteArray) String() string { + res := "{" + for i := int64(0); i < v.len; i++ { + if i > 0 { + res += ", " + } + res += v.elem(i).String() + } + return res + "}" +} + +func (v remoteArray) Assign(t *eval.Thread, o eval.Value) { + // TODO(austin) Could do a bigger memcpy if o is a + // remoteArray in the same Process. + oa := o.(eval.ArrayValue) + for i := int64(0); i < v.len; i++ { + v.Elem(t, i).Assign(t, oa.Elem(t, i)) + } +} + +func (v remoteArray) Get(t *eval.Thread) eval.ArrayValue { + return v +} + +func (v remoteArray) Elem(t *eval.Thread, i int64) eval.Value { + return v.elem(i) +} + +func (v remoteArray) elem(i int64) eval.Value { + return v.elemType.mk(v.r.plus(proc.Word(int64(v.elemType.size) * i))) +} + +func (v remoteArray) Sub(i int64, len int64) eval.ArrayValue { + return remoteArray{v.r.plus(proc.Word(int64(v.elemType.size) * i)), len, v.elemType} +} + +/* + * Struct + */ + +type remoteStruct struct { + r remote + layout []remoteStructField +} + +type remoteStructField struct { + offset int + fieldType *remoteType +} + +func (v remoteStruct) String() string { + res := "{" + for i := range v.layout { + if i > 0 { + res += ", " + } + res += v.field(i).String() + } + return res + "}" +} + +func (v remoteStruct) Assign(t *eval.Thread, o eval.Value) { + // TODO(austin) Could do a bigger memcpy. + oa := o.(eval.StructValue) + l := len(v.layout) + for i := 0; i < l; i++ { + v.Field(t, i).Assign(t, oa.Field(t, i)) + } +} + +func (v remoteStruct) Get(t *eval.Thread) eval.StructValue { + return v +} + +func (v remoteStruct) Field(t *eval.Thread, i int) eval.Value { + return v.field(i) +} + +func (v remoteStruct) field(i int) eval.Value { + f := &v.layout[i] + return f.fieldType.mk(v.r.plus(proc.Word(f.offset))) +} + +func (v remoteStruct) addr() remote { return v.r } + +/* + * Pointer + */ + +// TODO(austin) Comparing two remote pointers for equality in the +// interpreter will crash it because the Value's returned from +// remotePtr.Get() will be structs. + +type remotePtr struct { + r remote + elemType *remoteType +} + +func (v remotePtr) String() string { + return tryRVString(func(a aborter) string { + e := v.aGet(a) + if e == nil { + return "" + } + return "&" + e.String() + }) +} + +func (v remotePtr) Assign(t *eval.Thread, o eval.Value) { + v.Set(t, o.(eval.PtrValue).Get(t)) +} + +func (v remotePtr) Get(t *eval.Thread) eval.Value { + return v.aGet(t) +} + +func (v remotePtr) aGet(a aborter) eval.Value { + addr := proc.Word(v.r.Get(a, v.r.p.PtrSize())) + if addr == 0 { + return nil + } + return v.elemType.mk(remote{addr, v.r.p}) +} + +func (v remotePtr) Set(t *eval.Thread, x eval.Value) { + v.aSet(t, x) +} + +func (v remotePtr) aSet(a aborter, x eval.Value) { + if x == nil { + v.r.Set(a, v.r.p.PtrSize(), 0) + return + } + xr, ok := x.(remoteValue) + if !ok || v.r.p != xr.addr().p { + a.Abort(RemoteMismatchError("remote pointer must point within the same process")) + } + v.r.Set(a, v.r.p.PtrSize(), uint64(xr.addr().base)) +} + +func (v remotePtr) addr() remote { return v.r } + +/* + * Slice + */ + +type remoteSlice struct { + r remote + elemType *remoteType +} + +func (v remoteSlice) String() string { + return tryRVString(func(a aborter) string { + b := v.aGet(a).Base + if b == nil { + return "" + } + return b.String() + }) +} + +func (v remoteSlice) Assign(t *eval.Thread, o eval.Value) { + v.Set(t, o.(eval.SliceValue).Get(t)) +} + +func (v remoteSlice) Get(t *eval.Thread) eval.Slice { + return v.aGet(t) +} + +func (v remoteSlice) aGet(a aborter) eval.Slice { + rs := v.r.p.runtime.Slice.mk(v.r).(remoteStruct) + base := proc.Word(rs.field(v.r.p.f.Slice.Array).(remoteUint).aGet(a)) + nel := rs.field(v.r.p.f.Slice.Len).(remoteInt).aGet(a) + cap := rs.field(v.r.p.f.Slice.Cap).(remoteInt).aGet(a) + if base == 0 { + return eval.Slice{nil, nel, cap} + } + return eval.Slice{remoteArray{remote{base, v.r.p}, nel, v.elemType}, nel, cap} +} + +func (v remoteSlice) Set(t *eval.Thread, x eval.Slice) { + v.aSet(t, x) +} + +func (v remoteSlice) aSet(a aborter, x eval.Slice) { + rs := v.r.p.runtime.Slice.mk(v.r).(remoteStruct) + if x.Base == nil { + rs.field(v.r.p.f.Slice.Array).(remoteUint).aSet(a, 0) + } else { + ar, ok := x.Base.(remoteArray) + if !ok || v.r.p != ar.r.p { + a.Abort(RemoteMismatchError("remote slice must point within the same process")) + } + rs.field(v.r.p.f.Slice.Array).(remoteUint).aSet(a, uint64(ar.r.base)) + } + rs.field(v.r.p.f.Slice.Len).(remoteInt).aSet(a, x.Len) + rs.field(v.r.p.f.Slice.Cap).(remoteInt).aSet(a, x.Cap) +} diff --git a/libgo/go/exp/ogle/vars.go b/libgo/go/exp/ogle/vars.go new file mode 100644 index 000000000..8a3a14791 --- /dev/null +++ b/libgo/go/exp/ogle/vars.go @@ -0,0 +1,272 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ogle + +import ( + "debug/gosym" + "debug/proc" + "exp/eval" + "log" + "os" +) + +/* + * Remote frame pointers + */ + +// A NotOnStack error occurs when attempting to access a variable in a +// remote frame where that remote frame is not on the current stack. +type NotOnStack struct { + Fn *gosym.Func + Goroutine *Goroutine +} + +func (e NotOnStack) String() string { + return "function " + e.Fn.Name + " not on " + e.Goroutine.String() + "'s stack" +} + +// A remoteFramePtr is an implementation of eval.PtrValue that +// represents a pointer to a function frame in a remote process. When +// accessed, this locates the function on the current goroutine's +// stack and returns a structure containing the local variables of +// that function. +type remoteFramePtr struct { + p *Process + fn *gosym.Func + rt *remoteType +} + +func (v remoteFramePtr) String() string { + // TODO(austin): This could be a really awesome string method + return "" +} + +func (v remoteFramePtr) Assign(t *eval.Thread, o eval.Value) { + v.Set(t, o.(eval.PtrValue).Get(t)) +} + +func (v remoteFramePtr) Get(t *eval.Thread) eval.Value { + g := v.p.curGoroutine + if g == nil || g.frame == nil { + t.Abort(NoCurrentGoroutine{}) + } + + for f := g.frame; f != nil; f = f.aOuter(t) { + if f.fn != v.fn { + continue + } + + // TODO(austin): Register for shootdown with f + return v.rt.mk(remote{f.fp, v.p}) + } + + t.Abort(NotOnStack{v.fn, g}) + panic("fail") +} + +func (v remoteFramePtr) Set(t *eval.Thread, x eval.Value) { + // Theoretically this could be a static error. If remote + // packages were packages, remote frames could just be defined + // as constants. + t.Abort(ReadOnlyError("remote frames cannot be assigned to")) +} + +/* + * Remote packages + */ + +// TODO(austin): Remote packages are implemented as structs right now, +// which has some weird consequences. You can attempt to assign to a +// remote package. It also produces terrible error messages. +// Ideally, these would actually be packages, but somehow first-class +// so they could be assigned to other names. + +// A remotePackage is an implementation of eval.StructValue that +// represents a package in a remote process. It's essentially a +// regular struct, except it cannot be assigned to. +type remotePackage struct { + defs []eval.Value +} + +func (v remotePackage) String() string { return "" } + +func (v remotePackage) Assign(t *eval.Thread, o eval.Value) { + t.Abort(ReadOnlyError("remote packages cannot be assigned to")) +} + +func (v remotePackage) Get(t *eval.Thread) eval.StructValue { + return v +} + +func (v remotePackage) Field(t *eval.Thread, i int) eval.Value { + return v.defs[i] +} + +/* + * Remote variables + */ + +// populateWorld defines constants in the given world for each package +// in this process. These packages are structs that, in turn, contain +// fields for each global and function in that package. +func (p *Process) populateWorld(w *eval.World) os.Error { + type def struct { + t eval.Type + v eval.Value + } + packages := make(map[string]map[string]def) + + for _, s := range p.syms.Syms { + if s.ReceiverName() != "" { + // TODO(austin) + continue + } + + // Package + pkgName := s.PackageName() + switch pkgName { + case "", "type", "extratype", "string", "go": + // "go" is really "go.string" + continue + } + pkg, ok := packages[pkgName] + if !ok { + pkg = make(map[string]def) + packages[pkgName] = pkg + } + + // Symbol name + name := s.BaseName() + if _, ok := pkg[name]; ok { + log.Printf("Multiple definitions of symbol %s", s.Name) + continue + } + + // Symbol type + rt, err := p.typeOfSym(&s) + if err != nil { + return err + } + + // Definition + switch s.Type { + case 'D', 'd', 'B', 'b': + // Global variable + if rt == nil { + continue + } + pkg[name] = def{rt.Type, rt.mk(remote{proc.Word(s.Value), p})} + + case 'T', 't', 'L', 'l': + // Function + s := s.Func + // TODO(austin): Ideally, this would *also* be + // callable. How does that interact with type + // conversion syntax? + rt, err := p.makeFrameType(s) + if err != nil { + return err + } + pkg[name] = def{eval.NewPtrType(rt.Type), remoteFramePtr{p, s, rt}} + } + } + + // TODO(austin): Define remote types + + // Define packages + for pkgName, defs := range packages { + fields := make([]eval.StructField, len(defs)) + vals := make([]eval.Value, len(defs)) + i := 0 + for name, def := range defs { + fields[i].Name = name + fields[i].Type = def.t + vals[i] = def.v + i++ + } + pkgType := eval.NewStructType(fields) + pkgVal := remotePackage{vals} + + err := w.DefineConst(pkgName, pkgType, pkgVal) + if err != nil { + log.Printf("while defining package %s: %v", pkgName, err) + } + } + + return nil +} + +// typeOfSym returns the type associated with a symbol. If the symbol +// has no type, returns nil. +func (p *Process) typeOfSym(s *gosym.Sym) (*remoteType, os.Error) { + if s.GoType == 0 { + return nil, nil + } + addr := proc.Word(s.GoType) + var rt *remoteType + err := try(func(a aborter) { rt = parseRemoteType(a, p.runtime.Type.mk(remote{addr, p}).(remoteStruct)) }) + if err != nil { + return nil, err + } + return rt, nil +} + +// makeFrameType constructs a struct type for the frame of a function. +// The offsets in this struct type are such that the struct can be +// instantiated at this function's frame pointer. +func (p *Process) makeFrameType(s *gosym.Func) (*remoteType, os.Error) { + n := len(s.Params) + len(s.Locals) + fields := make([]eval.StructField, n) + layout := make([]remoteStructField, n) + i := 0 + + // TODO(austin): There can be multiple locals/parameters with + // the same name. We probably need liveness information to do + // anything about this. Once we have that, perhaps we give + // such fields interface{} type? Or perhaps we disambiguate + // the names with numbers. Disambiguation is annoying for + // things like "i", where there's an obvious right answer. + + for _, param := range s.Params { + rt, err := p.typeOfSym(param) + if err != nil { + return nil, err + } + if rt == nil { + //fmt.Printf(" (no type)\n"); + continue + } + // TODO(austin): Why do local variables carry their + // package name? + fields[i].Name = param.BaseName() + fields[i].Type = rt.Type + // Parameters have positive offsets from FP + layout[i].offset = int(param.Value) + layout[i].fieldType = rt + i++ + } + + for _, local := range s.Locals { + rt, err := p.typeOfSym(local) + if err != nil { + return nil, err + } + if rt == nil { + continue + } + fields[i].Name = local.BaseName() + fields[i].Type = rt.Type + // Locals have negative offsets from FP - PtrSize + layout[i].offset = -int(local.Value) - p.PtrSize() + layout[i].fieldType = rt + i++ + } + + fields = fields[0:i] + layout = layout[0:i] + t := eval.NewStructType(fields) + mk := func(r remote) eval.Value { return remoteStruct{r, layout} } + return &remoteType{t, 0, 0, mk}, nil +} diff --git a/libgo/go/expvar/expvar.go b/libgo/go/expvar/expvar.go new file mode 100644 index 000000000..b1f0f6c1b --- /dev/null +++ b/libgo/go/expvar/expvar.go @@ -0,0 +1,299 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The expvar package provides a standardized interface to public variables, +// such as operation counters in servers. It exposes these variables via +// HTTP at /debug/vars in JSON format. +// +// Operations to set or modify these public variables are atomic. +// +// In addition to adding the HTTP handler, this package registers the +// following variables: +// +// cmdline os.Args +// memstats runtime.Memstats +// +// The package is sometimes only imported for the side effect of +// registering its HTTP handler and the above variables. To use it +// this way, simply link this package into your program: +// import _ "expvar" +// +package expvar + +import ( + "bytes" + "fmt" + "http" + "json" + "log" + "os" + "runtime" + "strconv" + "sync" +) + +// Var is an abstract type for all exported variables. +type Var interface { + String() string +} + +// Int is a 64-bit integer variable that satisfies the Var interface. +type Int struct { + i int64 + mu sync.Mutex +} + +func (v *Int) String() string { return strconv.Itoa64(v.i) } + +func (v *Int) Add(delta int64) { + v.mu.Lock() + defer v.mu.Unlock() + v.i += delta +} + +func (v *Int) Set(value int64) { + v.mu.Lock() + defer v.mu.Unlock() + v.i = value +} + +// Float is a 64-bit float variable that satisfies the Var interface. +type Float struct { + f float64 + mu sync.Mutex +} + +func (v *Float) String() string { return strconv.Ftoa64(v.f, 'g', -1) } + +// Add adds delta to v. +func (v *Float) Add(delta float64) { + v.mu.Lock() + defer v.mu.Unlock() + v.f += delta +} + +// Set sets v to value. +func (v *Float) Set(value float64) { + v.mu.Lock() + defer v.mu.Unlock() + v.f = value +} + +// Map is a string-to-Var map variable that satisfies the Var interface. +type Map struct { + m map[string]Var + mu sync.Mutex +} + +// KeyValue represents a single entry in a Map. +type KeyValue struct { + Key string + Value Var +} + +func (v *Map) String() string { + v.mu.Lock() + defer v.mu.Unlock() + b := new(bytes.Buffer) + fmt.Fprintf(b, "{") + first := true + for key, val := range v.m { + if !first { + fmt.Fprintf(b, ", ") + } + fmt.Fprintf(b, "\"%s\": %v", key, val.String()) + first = false + } + fmt.Fprintf(b, "}") + return b.String() +} + +func (v *Map) Init() *Map { + v.m = make(map[string]Var) + return v +} + +func (v *Map) Get(key string) Var { + v.mu.Lock() + defer v.mu.Unlock() + return v.m[key] +} + +func (v *Map) Set(key string, av Var) { + v.mu.Lock() + defer v.mu.Unlock() + v.m[key] = av +} + +func (v *Map) Add(key string, delta int64) { + v.mu.Lock() + defer v.mu.Unlock() + av, ok := v.m[key] + if !ok { + av = new(Int) + v.m[key] = av + } + + // Add to Int; ignore otherwise. + if iv, ok := av.(*Int); ok { + iv.Add(delta) + } +} + +// AddFloat adds delta to the *Float value stored under the given map key. +func (v *Map) AddFloat(key string, delta float64) { + v.mu.Lock() + defer v.mu.Unlock() + av, ok := v.m[key] + if !ok { + av = new(Float) + v.m[key] = av + } + + // Add to Float; ignore otherwise. + if iv, ok := av.(*Float); ok { + iv.Add(delta) + } +} + +// TODO(rsc): Make sure map access in separate thread is safe. +func (v *Map) iterate(c chan<- KeyValue) { + for k, v := range v.m { + c <- KeyValue{k, v} + } + close(c) +} + +func (v *Map) Iter() <-chan KeyValue { + c := make(chan KeyValue) + go v.iterate(c) + return c +} + +// String is a string variable, and satisfies the Var interface. +type String struct { + s string +} + +func (v *String) String() string { return strconv.Quote(v.s) } + +func (v *String) Set(value string) { v.s = value } + +// IntFunc wraps a func() int64 to create a value that satisfies the Var interface. +// The function will be called each time the Var is evaluated. +type IntFunc func() int64 + +func (v IntFunc) String() string { return strconv.Itoa64(v()) } + +// FloatFunc wraps a func() float64 to create a value that satisfies the Var interface. +// The function will be called each time the Var is evaluated. +type FloatFunc func() float64 + +func (v FloatFunc) String() string { return strconv.Ftoa64(v(), 'g', -1) } + +// StringFunc wraps a func() string to create value that satisfies the Var interface. +// The function will be called each time the Var is evaluated. +type StringFunc func() string + +func (f StringFunc) String() string { return strconv.Quote(f()) } + + +// All published variables. +var vars map[string]Var = make(map[string]Var) +var mutex sync.Mutex + +// Publish declares an named exported variable. This should be called from a +// package's init function when it creates its Vars. If the name is already +// registered then this will log.Panic. +func Publish(name string, v Var) { + mutex.Lock() + defer mutex.Unlock() + if _, existing := vars[name]; existing { + log.Panicln("Reuse of exported var name:", name) + } + vars[name] = v +} + +// Get retrieves a named exported variable. +func Get(name string) Var { + return vars[name] +} + +// RemoveAll removes all exported variables. +// This is for tests; don't call this on a real server. +func RemoveAll() { + mutex.Lock() + defer mutex.Unlock() + vars = make(map[string]Var) +} + +// Convenience functions for creating new exported variables. + +func NewInt(name string) *Int { + v := new(Int) + Publish(name, v) + return v +} + +func NewFloat(name string) *Float { + v := new(Float) + Publish(name, v) + return v +} + +func NewMap(name string) *Map { + v := new(Map).Init() + Publish(name, v) + return v +} + +func NewString(name string) *String { + v := new(String) + Publish(name, v) + return v +} + +// TODO(rsc): Make sure map access in separate thread is safe. +func iterate(c chan<- KeyValue) { + for k, v := range vars { + c <- KeyValue{k, v} + } + close(c) +} + +func Iter() <-chan KeyValue { + c := make(chan KeyValue) + go iterate(c) + return c +} + +func expvarHandler(w http.ResponseWriter, r *http.Request) { + w.SetHeader("content-type", "application/json; charset=utf-8") + fmt.Fprintf(w, "{\n") + first := true + for name, value := range vars { + if !first { + fmt.Fprintf(w, ",\n") + } + first = false + fmt.Fprintf(w, "%q: %s", name, value) + } + fmt.Fprintf(w, "\n}\n") +} + +func memstats() string { + b, _ := json.MarshalIndent(&runtime.MemStats, "", "\t") + return string(b) +} + +func cmdline() string { + b, _ := json.Marshal(os.Args) + return string(b) +} + +func init() { + http.Handle("/debug/vars", http.HandlerFunc(expvarHandler)) + Publish("cmdline", StringFunc(cmdline)) + Publish("memstats", StringFunc(memstats)) +} diff --git a/libgo/go/expvar/expvar_test.go b/libgo/go/expvar/expvar_test.go new file mode 100644 index 000000000..a8b1a96a9 --- /dev/null +++ b/libgo/go/expvar/expvar_test.go @@ -0,0 +1,154 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package expvar + +import ( + "json" + "testing" +) + +func TestInt(t *testing.T) { + reqs := NewInt("requests") + if reqs.i != 0 { + t.Errorf("reqs.i = %v, want 0", reqs.i) + } + if reqs != Get("requests").(*Int) { + t.Errorf("Get() failed.") + } + + reqs.Add(1) + reqs.Add(3) + if reqs.i != 4 { + t.Errorf("reqs.i = %v, want 4", reqs.i) + } + + if s := reqs.String(); s != "4" { + t.Errorf("reqs.String() = %q, want \"4\"", s) + } + + reqs.Set(-2) + if reqs.i != -2 { + t.Errorf("reqs.i = %v, want -2", reqs.i) + } +} + +func TestFloat(t *testing.T) { + reqs := NewFloat("requests-float") + if reqs.f != 0.0 { + t.Errorf("reqs.f = %v, want 0", reqs.f) + } + if reqs != Get("requests-float").(*Float) { + t.Errorf("Get() failed.") + } + + reqs.Add(1.5) + reqs.Add(1.25) + if reqs.f != 2.75 { + t.Errorf("reqs.f = %v, want 2.75", reqs.f) + } + + if s := reqs.String(); s != "2.75" { + t.Errorf("reqs.String() = %q, want \"4.64\"", s) + } + + reqs.Add(-2) + if reqs.f != 0.75 { + t.Errorf("reqs.f = %v, want 0.75", reqs.f) + } +} + +func TestString(t *testing.T) { + name := NewString("my-name") + if name.s != "" { + t.Errorf("name.s = %q, want \"\"", name.s) + } + + name.Set("Mike") + if name.s != "Mike" { + t.Errorf("name.s = %q, want \"Mike\"", name.s) + } + + if s := name.String(); s != "\"Mike\"" { + t.Errorf("reqs.String() = %q, want \"\"Mike\"\"", s) + } +} + +func TestMapCounter(t *testing.T) { + colours := NewMap("bike-shed-colours") + + colours.Add("red", 1) + colours.Add("red", 2) + colours.Add("blue", 4) + colours.AddFloat("green", 4.125) + if x := colours.m["red"].(*Int).i; x != 3 { + t.Errorf("colours.m[\"red\"] = %v, want 3", x) + } + if x := colours.m["blue"].(*Int).i; x != 4 { + t.Errorf("colours.m[\"blue\"] = %v, want 4", x) + } + if x := colours.m["green"].(*Float).f; x != 4.125 { + t.Errorf("colours.m[\"green\"] = %v, want 3.14", x) + } + + // colours.String() should be '{"red":3, "blue":4}', + // though the order of red and blue could vary. + s := colours.String() + var j interface{} + err := json.Unmarshal([]byte(s), &j) + if err != nil { + t.Errorf("colours.String() isn't valid JSON: %v", err) + } + m, ok := j.(map[string]interface{}) + if !ok { + t.Error("colours.String() didn't produce a map.") + } + red := m["red"] + x, ok := red.(float64) + if !ok { + t.Error("red.Kind() is not a number.") + } + if x != 3 { + t.Errorf("red = %v, want 3", x) + } +} + +func TestIntFunc(t *testing.T) { + x := int64(4) + ix := IntFunc(func() int64 { return x }) + if s := ix.String(); s != "4" { + t.Errorf("ix.String() = %v, want 4", s) + } + + x++ + if s := ix.String(); s != "5" { + t.Errorf("ix.String() = %v, want 5", s) + } +} + +func TestFloatFunc(t *testing.T) { + x := 8.5 + ix := FloatFunc(func() float64 { return x }) + if s := ix.String(); s != "8.5" { + t.Errorf("ix.String() = %v, want 3.14", s) + } + + x -= 1.25 + if s := ix.String(); s != "7.25" { + t.Errorf("ix.String() = %v, want 4.34", s) + } +} + +func TestStringFunc(t *testing.T) { + x := "hello" + sx := StringFunc(func() string { return x }) + if s, exp := sx.String(), `"hello"`; s != exp { + t.Errorf(`sx.String() = %q, want %q`, s, exp) + } + + x = "goodbye" + if s, exp := sx.String(), `"goodbye"`; s != exp { + t.Errorf(`sx.String() = %q, want %q`, s, exp) + } +} diff --git a/libgo/go/flag/export_test.go b/libgo/go/flag/export_test.go new file mode 100644 index 000000000..b5e3243b3 --- /dev/null +++ b/libgo/go/flag/export_test.go @@ -0,0 +1,32 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package flag + +import "os" + +// Additional routines compiled into the package only during testing. + +// ResetForTesting clears all flag state and sets the usage function as directed. +// After calling ResetForTesting, parse errors in flag handling will panic rather +// than exit the program. +func ResetForTesting(usage func()) { + flags = &allFlags{make(map[string]*Flag), make(map[string]*Flag), os.Args[1:]} + Usage = usage + panicOnError = true +} + +// ParseForTesting parses the flag state using the provided arguments. It +// should be called after 1) ResetForTesting and 2) setting up the new flags. +// The return value reports whether the parse was error-free. +func ParseForTesting(args []string) (result bool) { + defer func() { + if recover() != nil { + result = false + } + }() + os.Args = args + Parse() + return true +} diff --git a/libgo/go/flag/flag.go b/libgo/go/flag/flag.go new file mode 100644 index 000000000..143a10611 --- /dev/null +++ b/libgo/go/flag/flag.go @@ -0,0 +1,480 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + The flag package implements command-line flag parsing. + + Usage: + + Define flags using flag.String(), Bool(), Int(), etc. Example: + import "flag" + var ip *int = flag.Int("flagname", 1234, "help message for flagname") + If you like, you can bind the flag to a variable using the Var() functions. + var flagvar int + func init() { + flag.IntVar(&flagvar, "flagname", 1234, "help message for flagname") + } + Or you can create custom flags that satisfy the Value interface (with + pointer receivers) and couple them to flag parsing by + flag.Var(&flagVal, "name", "help message for flagname") + For such flags, the default value is just the initial value of the variable. + + After all flags are defined, call + flag.Parse() + to parse the command line into the defined flags. + + Flags may then be used directly. If you're using the flags themselves, + they are all pointers; if you bind to variables, they're values. + fmt.Println("ip has value ", *ip); + fmt.Println("flagvar has value ", flagvar); + + After parsing, the arguments after the flag are available as the + slice flag.Args() or individually as flag.Arg(i). + The arguments are indexed from 0 up to flag.NArg(). + + Command line flag syntax: + -flag + -flag=x + -flag x // non-boolean flags only + One or two minus signs may be used; they are equivalent. + The last form is not permitted for boolean flags because the + meaning of the command + cmd -x * + will change if there is a file called 0, false, etc. You must + use the -flag=false form to turn off a boolean flag. + + Flag parsing stops just before the first non-flag argument + ("-" is a non-flag argument) or after the terminator "--". + + Integer flags accept 1234, 0664, 0x1234 and may be negative. + Boolean flags may be 1, 0, t, f, true, false, TRUE, FALSE, True, False. + + It is safe to call flag.Parse multiple times, possibly after changing + os.Args. This makes it possible to implement command lines with + subcommands that enable additional flags, as in: + + flag.Bool(...) // global options + flag.Parse() // parse leading command + subcmd := flag.Args(0) + switch subcmd { + // add per-subcommand options + } + os.Args = flag.Args() + flag.Parse() +*/ +package flag + +import ( + "fmt" + "os" + "strconv" +) + +// -- Bool Value +type boolValue bool + +func newBoolValue(val bool, p *bool) *boolValue { + *p = val + return (*boolValue)(p) +} + +func (b *boolValue) Set(s string) bool { + v, err := strconv.Atob(s) + *b = boolValue(v) + return err == nil +} + +func (b *boolValue) String() string { return fmt.Sprintf("%v", *b) } + +// -- Int Value +type intValue int + +func newIntValue(val int, p *int) *intValue { + *p = val + return (*intValue)(p) +} + +func (i *intValue) Set(s string) bool { + v, err := strconv.Atoi(s) + *i = intValue(v) + return err == nil +} + +func (i *intValue) String() string { return fmt.Sprintf("%v", *i) } + +// -- Int64 Value +type int64Value int64 + +func newInt64Value(val int64, p *int64) *int64Value { + *p = val + return (*int64Value)(p) +} + +func (i *int64Value) Set(s string) bool { + v, err := strconv.Atoi64(s) + *i = int64Value(v) + return err == nil +} + +func (i *int64Value) String() string { return fmt.Sprintf("%v", *i) } + +// -- Uint Value +type uintValue uint + +func newUintValue(val uint, p *uint) *uintValue { + *p = val + return (*uintValue)(p) +} + +func (i *uintValue) Set(s string) bool { + v, err := strconv.Atoui(s) + *i = uintValue(v) + return err == nil +} + +func (i *uintValue) String() string { return fmt.Sprintf("%v", *i) } + +// -- uint64 Value +type uint64Value uint64 + +func newUint64Value(val uint64, p *uint64) *uint64Value { + *p = val + return (*uint64Value)(p) +} + +func (i *uint64Value) Set(s string) bool { + v, err := strconv.Atoui64(s) + *i = uint64Value(v) + return err == nil +} + +func (i *uint64Value) String() string { return fmt.Sprintf("%v", *i) } + +// -- string Value +type stringValue string + +func newStringValue(val string, p *string) *stringValue { + *p = val + return (*stringValue)(p) +} + +func (s *stringValue) Set(val string) bool { + *s = stringValue(val) + return true +} + +func (s *stringValue) String() string { return fmt.Sprintf("%s", *s) } + +// -- Float64 Value +type float64Value float64 + +func newFloat64Value(val float64, p *float64) *float64Value { + *p = val + return (*float64Value)(p) +} + +func (f *float64Value) Set(s string) bool { + v, err := strconv.Atof64(s) + *f = float64Value(v) + return err == nil +} + +func (f *float64Value) String() string { return fmt.Sprintf("%v", *f) } + +// Value is the interface to the dynamic value stored in a flag. +// (The default value is represented as a string.) +type Value interface { + String() string + Set(string) bool +} + +// A Flag represents the state of a flag. +type Flag struct { + Name string // name as it appears on command line + Usage string // help message + Value Value // value as set + DefValue string // default value (as text); for usage message +} + +type allFlags struct { + actual map[string]*Flag + formal map[string]*Flag + args []string // arguments after flags +} + +var flags *allFlags + +// VisitAll visits the flags, calling fn for each. It visits all flags, even those not set. +func VisitAll(fn func(*Flag)) { + for _, f := range flags.formal { + fn(f) + } +} + +// Visit visits the flags, calling fn for each. It visits only those flags that have been set. +func Visit(fn func(*Flag)) { + for _, f := range flags.actual { + fn(f) + } +} + +// Lookup returns the Flag structure of the named flag, returning nil if none exists. +func Lookup(name string) *Flag { + return flags.formal[name] +} + +// Set sets the value of the named flag. It returns true if the set succeeded; false if +// there is no such flag defined. +func Set(name, value string) bool { + f, ok := flags.formal[name] + if !ok { + return false + } + ok = f.Value.Set(value) + if !ok { + return false + } + flags.actual[name] = f + return true +} + +// PrintDefaults prints to standard error the default values of all defined flags. +func PrintDefaults() { + VisitAll(func(f *Flag) { + format := " -%s=%s: %s\n" + if _, ok := f.Value.(*stringValue); ok { + // put quotes on the value + format = " -%s=%q: %s\n" + } + fmt.Fprintf(os.Stderr, format, f.Name, f.DefValue, f.Usage) + }) +} + +// Usage prints to standard error a default usage message documenting all defined flags. +// The function is a variable that may be changed to point to a custom function. +var Usage = func() { + fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0]) + PrintDefaults() +} + +var panicOnError = false + +func fail() { + Usage() + if panicOnError { + panic("flag parse error") + } + os.Exit(2) +} + +func NFlag() int { return len(flags.actual) } + +// Arg returns the i'th command-line argument. Arg(0) is the first remaining argument +// after flags have been processed. +func Arg(i int) string { + if i < 0 || i >= len(flags.args) { + return "" + } + return flags.args[i] +} + +// NArg is the number of arguments remaining after flags have been processed. +func NArg() int { return len(flags.args) } + +// Args returns the non-flag command-line arguments. +func Args() []string { return flags.args } + +// BoolVar defines a bool flag with specified name, default value, and usage string. +// The argument p points to a bool variable in which to store the value of the flag. +func BoolVar(p *bool, name string, value bool, usage string) { + Var(newBoolValue(value, p), name, usage) +} + +// Bool defines a bool flag with specified name, default value, and usage string. +// The return value is the address of a bool variable that stores the value of the flag. +func Bool(name string, value bool, usage string) *bool { + p := new(bool) + BoolVar(p, name, value, usage) + return p +} + +// IntVar defines an int flag with specified name, default value, and usage string. +// The argument p points to an int variable in which to store the value of the flag. +func IntVar(p *int, name string, value int, usage string) { + Var(newIntValue(value, p), name, usage) +} + +// Int defines an int flag with specified name, default value, and usage string. +// The return value is the address of an int variable that stores the value of the flag. +func Int(name string, value int, usage string) *int { + p := new(int) + IntVar(p, name, value, usage) + return p +} + +// Int64Var defines an int64 flag with specified name, default value, and usage string. +// The argument p points to an int64 variable in which to store the value of the flag. +func Int64Var(p *int64, name string, value int64, usage string) { + Var(newInt64Value(value, p), name, usage) +} + +// Int64 defines an int64 flag with specified name, default value, and usage string. +// The return value is the address of an int64 variable that stores the value of the flag. +func Int64(name string, value int64, usage string) *int64 { + p := new(int64) + Int64Var(p, name, value, usage) + return p +} + +// UintVar defines a uint flag with specified name, default value, and usage string. +// The argument p points to a uint variable in which to store the value of the flag. +func UintVar(p *uint, name string, value uint, usage string) { + Var(newUintValue(value, p), name, usage) +} + +// Uint defines a uint flag with specified name, default value, and usage string. +// The return value is the address of a uint variable that stores the value of the flag. +func Uint(name string, value uint, usage string) *uint { + p := new(uint) + UintVar(p, name, value, usage) + return p +} + +// Uint64Var defines a uint64 flag with specified name, default value, and usage string. +// The argument p points to a uint64 variable in which to store the value of the flag. +func Uint64Var(p *uint64, name string, value uint64, usage string) { + Var(newUint64Value(value, p), name, usage) +} + +// Uint64 defines a uint64 flag with specified name, default value, and usage string. +// The return value is the address of a uint64 variable that stores the value of the flag. +func Uint64(name string, value uint64, usage string) *uint64 { + p := new(uint64) + Uint64Var(p, name, value, usage) + return p +} + +// StringVar defines a string flag with specified name, default value, and usage string. +// The argument p points to a string variable in which to store the value of the flag. +func StringVar(p *string, name, value string, usage string) { + Var(newStringValue(value, p), name, usage) +} + +// String defines a string flag with specified name, default value, and usage string. +// The return value is the address of a string variable that stores the value of the flag. +func String(name, value string, usage string) *string { + p := new(string) + StringVar(p, name, value, usage) + return p +} + +// Float64Var defines a float64 flag with specified name, default value, and usage string. +// The argument p points to a float64 variable in which to store the value of the flag. +func Float64Var(p *float64, name string, value float64, usage string) { + Var(newFloat64Value(value, p), name, usage) +} + +// Float64 defines a float64 flag with specified name, default value, and usage string. +// The return value is the address of a float64 variable that stores the value of the flag. +func Float64(name string, value float64, usage string) *float64 { + p := new(float64) + Float64Var(p, name, value, usage) + return p +} + +// Var defines a user-typed flag with specified name, default value, and usage string. +// The argument p points to a Value variable in which to store the value of the flag. +func Var(value Value, name string, usage string) { + // Remember the default value as a string; it won't change. + f := &Flag{name, usage, value, value.String()} + _, alreadythere := flags.formal[name] + if alreadythere { + fmt.Fprintln(os.Stderr, "flag redefined:", name) + panic("flag redefinition") // Happens only if flags are declared with identical names + } + flags.formal[name] = f +} + + +func (f *allFlags) parseOne() (ok bool) { + if len(f.args) == 0 { + return false + } + s := f.args[0] + if len(s) == 0 || s[0] != '-' || len(s) == 1 { + return false + } + num_minuses := 1 + if s[1] == '-' { + num_minuses++ + if len(s) == 2 { // "--" terminates the flags + f.args = f.args[1:] + return false + } + } + name := s[num_minuses:] + if len(name) == 0 || name[0] == '-' || name[0] == '=' { + fmt.Fprintln(os.Stderr, "bad flag syntax:", s) + fail() + } + + // it's a flag. does it have an argument? + f.args = f.args[1:] + has_value := false + value := "" + for i := 1; i < len(name); i++ { // equals cannot be first + if name[i] == '=' { + value = name[i+1:] + has_value = true + name = name[0:i] + break + } + } + m := flags.formal + flag, alreadythere := m[name] // BUG + if !alreadythere { + fmt.Fprintf(os.Stderr, "flag provided but not defined: -%s\n", name) + fail() + } + if fv, ok := flag.Value.(*boolValue); ok { // special case: doesn't need an arg + if has_value { + if !fv.Set(value) { + fmt.Fprintf(os.Stderr, "invalid boolean value %q for flag: -%s\n", value, name) + fail() + } + } else { + fv.Set("true") + } + } else { + // It must have a value, which might be the next argument. + if !has_value && len(f.args) > 0 { + // value is the next arg + has_value = true + value, f.args = f.args[0], f.args[1:] + } + if !has_value { + fmt.Fprintf(os.Stderr, "flag needs an argument: -%s\n", name) + fail() + } + ok = flag.Value.Set(value) + if !ok { + fmt.Fprintf(os.Stderr, "invalid value %q for flag: -%s\n", value, name) + fail() + } + } + flags.actual[name] = flag + return true +} + +// Parse parses the command-line flags. Must be called after all flags are defined +// and before any are accessed by the program. +func Parse() { + flags.args = os.Args[1:] + for flags.parseOne() { + } +} + +func init() { + flags = &allFlags{make(map[string]*Flag), make(map[string]*Flag), os.Args[1:]} +} diff --git a/libgo/go/flag/flag_test.go b/libgo/go/flag/flag_test.go new file mode 100644 index 000000000..b91a8b567 --- /dev/null +++ b/libgo/go/flag/flag_test.go @@ -0,0 +1,194 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package flag_test + +import ( + . "flag" + "fmt" + "os" + "testing" +) + +var ( + test_bool = Bool("test_bool", false, "bool value") + test_int = Int("test_int", 0, "int value") + test_int64 = Int64("test_int64", 0, "int64 value") + test_uint = Uint("test_uint", 0, "uint value") + test_uint64 = Uint64("test_uint64", 0, "uint64 value") + test_string = String("test_string", "0", "string value") + test_float64 = Float64("test_float64", 0, "float64 value") +) + +func boolString(s string) string { + if s == "0" { + return "false" + } + return "true" +} + +func TestEverything(t *testing.T) { + m := make(map[string]*Flag) + desired := "0" + visitor := func(f *Flag) { + if len(f.Name) > 5 && f.Name[0:5] == "test_" { + m[f.Name] = f + ok := false + switch { + case f.Value.String() == desired: + ok = true + case f.Name == "test_bool" && f.Value.String() == boolString(desired): + ok = true + } + if !ok { + t.Error("Visit: bad value", f.Value.String(), "for", f.Name) + } + } + } + VisitAll(visitor) + if len(m) != 7 { + t.Error("VisitAll misses some flags") + for k, v := range m { + t.Log(k, *v) + } + } + m = make(map[string]*Flag) + Visit(visitor) + if len(m) != 0 { + t.Errorf("Visit sees unset flags") + for k, v := range m { + t.Log(k, *v) + } + } + // Now set all flags + Set("test_bool", "true") + Set("test_int", "1") + Set("test_int64", "1") + Set("test_uint", "1") + Set("test_uint64", "1") + Set("test_string", "1") + Set("test_float64", "1") + desired = "1" + Visit(visitor) + if len(m) != 7 { + t.Error("Visit fails after set") + for k, v := range m { + t.Log(k, *v) + } + } +} + +func TestUsage(t *testing.T) { + called := false + ResetForTesting(func() { called = true }) + if ParseForTesting([]string{"a.out", "-x"}) { + t.Error("parse did not fail for unknown flag") + } + if !called { + t.Error("did not call Usage for unknown flag") + } +} + +func TestParse(t *testing.T) { + ResetForTesting(func() { t.Error("bad parse") }) + boolFlag := Bool("bool", false, "bool value") + bool2Flag := Bool("bool2", false, "bool2 value") + intFlag := Int("int", 0, "int value") + int64Flag := Int64("int64", 0, "int64 value") + uintFlag := Uint("uint", 0, "uint value") + uint64Flag := Uint64("uint64", 0, "uint64 value") + stringFlag := String("string", "0", "string value") + float64Flag := Float64("float64", 0, "float64 value") + extra := "one-extra-argument" + args := []string{ + "a.out", + "-bool", + "-bool2=true", + "--int", "22", + "--int64", "23", + "-uint", "24", + "--uint64", "25", + "-string", "hello", + "-float64", "2718e28", + extra, + } + if !ParseForTesting(args) { + t.Fatal("parse failed") + } + if *boolFlag != true { + t.Error("bool flag should be true, is ", *boolFlag) + } + if *bool2Flag != true { + t.Error("bool2 flag should be true, is ", *bool2Flag) + } + if *intFlag != 22 { + t.Error("int flag should be 22, is ", *intFlag) + } + if *int64Flag != 23 { + t.Error("int64 flag should be 23, is ", *int64Flag) + } + if *uintFlag != 24 { + t.Error("uint flag should be 24, is ", *uintFlag) + } + if *uint64Flag != 25 { + t.Error("uint64 flag should be 25, is ", *uint64Flag) + } + if *stringFlag != "hello" { + t.Error("string flag should be `hello`, is ", *stringFlag) + } + if *float64Flag != 2718e28 { + t.Error("float64 flag should be 2718e28, is ", *float64Flag) + } + if len(Args()) != 1 { + t.Error("expected one argument, got", len(Args())) + } else if Args()[0] != extra { + t.Errorf("expected argument %q got %q", extra, Args()[0]) + } +} + +// Declare a user-defined flag. +type flagVar []string + +func (f *flagVar) String() string { + return fmt.Sprint([]string(*f)) +} + +func (f *flagVar) Set(value string) bool { + *f = append(*f, value) + return true +} + +func TestUserDefined(t *testing.T) { + ResetForTesting(func() { t.Fatal("bad parse") }) + var v flagVar + Var(&v, "v", "usage") + if !ParseForTesting([]string{"a.out", "-v", "1", "-v", "2", "-v=3"}) { + t.Error("parse failed") + } + if len(v) != 3 { + t.Fatal("expected 3 args; got ", len(v)) + } + expect := "[1 2 3]" + if v.String() != expect { + t.Errorf("expected value %q got %q", expect, v.String()) + } +} + +func TestChangingArgs(t *testing.T) { + ResetForTesting(func() { t.Fatal("bad parse") }) + oldArgs := os.Args + defer func() { os.Args = oldArgs }() + os.Args = []string{"cmd", "-before", "subcmd", "-after", "args"} + before := Bool("before", false, "") + Parse() + cmd := Arg(0) + os.Args = Args() + after := Bool("after", false, "") + Parse() + args := Args() + + if !*before || cmd != "subcmd" || !*after || len(args) != 1 || args[0] != "args" { + t.Fatalf("expected true subcmd true [args] got %v %v %v %v", *before, cmd, *after, args) + } +} diff --git a/libgo/go/fmt/doc.go b/libgo/go/fmt/doc.go new file mode 100644 index 000000000..191bf68b1 --- /dev/null +++ b/libgo/go/fmt/doc.go @@ -0,0 +1,164 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + Package fmt implements formatted I/O with functions analogous + to C's printf and scanf. The format 'verbs' are derived from C's but + are simpler. + + Printing: + + The verbs: + + General: + %v the value in a default format. + when printing structs, the plus flag (%+v) adds field names + %#v a Go-syntax representation of the value + %T a Go-syntax representation of the type of the value + + Boolean: + %t the word true or false + Integer: + %b base 2 + %c the character represented by the corresponding Unicode code point + %d base 10 + %o base 8 + %x base 16, with lower-case letters for a-f + %X base 16, with upper-case letters for A-F + %U unicode format: U+1234; same as "U+%x" with 4 digits default + Floating-point and complex constituents: + %e scientific notation, e.g. -1234.456e+78 + %E scientific notation, e.g. -1234.456E+78 + %f decimal point but no exponent, e.g. 123.456 + %g whichever of %e or %f produces more compact output + %G whichever of %E or %f produces more compact output + String and slice of bytes: + %s the uninterpreted bytes of the string or slice + %q a double-quoted string safely escaped with Go syntax + %x base 16, lower-case, two characters per byte + %X base 16, upper-case, two characters per byte + Pointer: + %p base 16 notation, with leading 0x + + There is no 'u' flag. Integers are printed unsigned if they have unsigned type. + Similarly, there is no need to specify the size of the operand (int8, int64). + + For numeric values, the width and precision flags control + formatting; width sets the width of the field, precision the + number of places after the decimal, if appropriate. The + format %6.2f prints 123.45. The width of a field is the number + of Unicode code points in the string. This differs from C's printf where + the field width is the number of bytes. Either or both of the + flags may be replaced with the character '*', causing their values + to be obtained from the next operand, which must be of type int. + + Other flags: + + always print a sign for numeric values + - pad with spaces on the right rather than the left (left-justify the field) + # alternate format: add leading 0 for octal (%#o), 0x for hex (%#x); + 0X for hex (%#X); suppress 0x for %p (%#p); + print a raw (backquoted) string if possible for %q (%#q) + ' ' (space) leave a space for elided sign in numbers (% d); + put spaces between bytes printing strings or slices in hex (% x, % X) + 0 pad with leading zeros rather than spaces + + For each Printf-like function, there is also a Print function + that takes no format and is equivalent to saying %v for every + operand. Another variant Println inserts blanks between + operands and appends a newline. + + Regardless of the verb, if an operand is an interface value, + the internal concrete value is used, not the interface itself. + Thus: + var i interface{} = 23 + fmt.Printf("%v\n", i) + will print 23. + + If an operand implements interface Formatter, that interface + can be used for fine control of formatting. + + If an operand implements method String() string that method + will be used to convert the object to a string, which will then + be formatted as required by the verb (if any). To avoid + recursion in cases such as + type X int + func (x X) String() string { return Sprintf("%d", x) } + cast the value before recurring: + func (x X) String() string { return Sprintf("%d", int(x)) } + + Format errors: + + If an invalid argument is given for a verb, such as providing + a string to %d, the generated string will contain a + description of the problem, as in these examples: + + Wrong type or unknown verb: %!verb(type=value) + Printf("%d", hi): %!d(string=hi) + Too many arguments: %!(EXTRA type=value) + Printf("hi", "guys"): hi%!(EXTRA string=guys) + Too few arguments: %!verb(MISSING) + Printf("hi%d"): hi %!d(MISSING) + Non-int for width or precision: %!(BADWIDTH) or %!(BADPREC) + Printf("%*s", 4.5, "hi"): %!(BADWIDTH)hi + Printf("%.*s", 4.5, "hi"): %!(BADPREC)hi + + All errors begin with the string "%!" followed sometimes + by a single character (the verb) and end with a parenthesized + description. + + Scanning: + + An analogous set of functions scans formatted text to yield + values. Scan, Scanf and Scanln read from os.Stdin; Fscan, + Fscanf and Fscanln read from a specified os.Reader; Sscan, + Sscanf and Sscanln read from an argument string. Sscanln, + Fscanln and Sscanln stop scanning at a newline and require that + the items be followed by one; Sscanf, Fscanf and Sscanf require + newlines in the input to match newlines in the format; the other + routines treat newlines as spaces. + + Scanf, Fscanf, and Sscanf parse the arguments according to a + format string, analogous to that of Printf. For example, %x + will scan an integer as a hexadecimal number, and %v will scan + the default representation format for the value. + + The formats behave analogously to those of Printf with the + following exceptions: + + %p is not implemented + %T is not implemented + %e %E %f %F %g %g are all equivalent and scan any floating point or complex value + %s and %v on strings scan a space-delimited token + + Width is interpreted in the input text (%5s means at most + five runes of input will be read to scan a string) but there + is no syntax for scanning with a precision (no %5.2f, just + %5f). + + When scanning with a format, all non-empty runs of space + characters (except newline) are equivalent to a single + space in both the format and the input. With that proviso, + text in the format string must match the input text; scanning + stops if it does not, with the return value of the function + indicating the number of arguments scanned. + + In all the scanning functions, if an operand implements method + Scan (that is, it implements the Scanner interface) that + method will be used to scan the text for that operand. Also, + if the number of arguments scanned is less than the number of + arguments provided, an error is returned. + + All arguments to be scanned must be either pointers to basic + types or implementations of the Scanner interface. + + Note: Fscan etc. can read one character (rune) past the + input they return, which means that a loop calling a scan + routine may skip some of the input. This is usually a + problem only when there is no space between input values. + However, if the reader provided to Fscan implements UnreadRune, + that method will be used to save the character and successive + calls will not lose data. To attach an UnreadRune method + to a reader without that capability, use bufio.NewReader. +*/ +package fmt diff --git a/libgo/go/fmt/fmt_test.go b/libgo/go/fmt/fmt_test.go new file mode 100644 index 000000000..3f085b722 --- /dev/null +++ b/libgo/go/fmt/fmt_test.go @@ -0,0 +1,650 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fmt_test + +import ( + . "fmt" + "io" + "math" + "runtime" // for the malloc count test only + "strings" + "testing" +) + +type ( + renamedBool bool + renamedInt int + renamedInt8 int8 + renamedInt16 int16 + renamedInt32 int32 + renamedInt64 int64 + renamedUint uint + renamedUint8 uint8 + renamedUint16 uint16 + renamedUint32 uint32 + renamedUint64 uint64 + renamedUintptr uintptr + renamedString string + renamedBytes []byte + renamedFloat32 float32 + renamedFloat64 float64 + renamedComplex64 complex64 + renamedComplex128 complex128 +) + +func TestFmtInterface(t *testing.T) { + var i1 interface{} + i1 = "abc" + s := Sprintf("%s", i1) + if s != "abc" { + t.Errorf(`Sprintf("%%s", empty("abc")) = %q want %q`, s, "abc") + } +} + + +const b32 uint32 = 1<<32 - 1 +const b64 uint64 = 1<<64 - 1 + +var array = []int{1, 2, 3, 4, 5} +var iarray = []interface{}{1, "hello", 2.5, nil} + +type A struct { + i int + j uint + s string + x []int +} + +type I int + +func (i I) String() string { return Sprintf("<%d>", int(i)) } + +type B struct { + i I + j int +} + +type C struct { + i int + B +} + +type F int + +func (f F) Format(s State, c int) { + Fprintf(s, "<%c=F(%d)>", c, int(f)) +} + +type G int + +func (g G) GoString() string { + return Sprintf("GoString(%d)", int(g)) +} + +type S struct { + f F // a struct field that Formats + g G // a struct field that GoStrings +} + +// A type with a String method with pointer receiver for testing %p +type P int + +var pValue P + +func (p *P) String() string { + return "String(p)" +} + +var b byte + +var fmttests = []struct { + fmt string + val interface{} + out string +}{ + {"%d", 12345, "12345"}, + {"%v", 12345, "12345"}, + {"%t", true, "true"}, + + // basic string + {"%s", "abc", "abc"}, + {"%x", "abc", "616263"}, + {"%x", "xyz", "78797a"}, + {"%X", "xyz", "78797A"}, + {"%q", "abc", `"abc"`}, + + // basic bytes + {"%s", []byte("abc"), "abc"}, + {"%x", []byte("abc"), "616263"}, + {"% x", []byte("abc\xff"), "61 62 63 ff"}, + {"% X", []byte("abc\xff"), "61 62 63 FF"}, + {"%x", []byte("xyz"), "78797a"}, + {"%X", []byte("xyz"), "78797A"}, + {"%q", []byte("abc"), `"abc"`}, + + // escaped strings + {"%#q", `abc`, "`abc`"}, + {"%#q", `"`, "`\"`"}, + {"1 %#q", `\n`, "1 `\\n`"}, + {"2 %#q", "\n", `2 "\n"`}, + {"%q", `"`, `"\""`}, + {"%q", "\a\b\f\r\n\t\v", `"\a\b\f\r\n\t\v"`}, + {"%q", "abc\xffdef", `"abc\xffdef"`}, + {"%q", "\u263a", `"\u263a"`}, + {"%q", "\U0010ffff", `"\U0010ffff"`}, + + // width + {"%5s", "abc", " abc"}, + {"%2s", "\u263a", " \u263a"}, + {"%-5s", "abc", "abc "}, + {"%05s", "abc", "00abc"}, + + // integers + {"%d", 12345, "12345"}, + {"%d", -12345, "-12345"}, + {"%10d", 12345, " 12345"}, + {"%10d", -12345, " -12345"}, + {"%+10d", 12345, " +12345"}, + {"%010d", 12345, "0000012345"}, + {"%010d", -12345, "-000012345"}, + {"%-10d", 12345, "12345 "}, + {"%010.3d", 1, " 001"}, + {"%010.3d", -1, " -001"}, + {"%+d", 12345, "+12345"}, + {"%+d", -12345, "-12345"}, + {"%+d", 0, "+0"}, + {"% d", 0, " 0"}, + {"% d", 12345, " 12345"}, + + // unicode format + {"%U", 0x1, "U+0001"}, + {"%.8U", 0x2, "U+00000002"}, + {"%U", 0x1234, "U+1234"}, + {"%U", 0x12345, "U+12345"}, + {"%10.6U", 0xABC, " U+000ABC"}, + {"%-10.6U", 0xABC, "U+000ABC "}, + + // floats + {"%+.3e", 0.0, "+0.000e+00"}, + {"%+.3e", 1.0, "+1.000e+00"}, + {"%+.3f", -1.0, "-1.000"}, + {"% .3E", -1.0, "-1.000E+00"}, + {"% .3e", 1.0, " 1.000e+00"}, + {"%+.3g", 0.0, "+0"}, + {"%+.3g", 1.0, "+1"}, + {"%+.3g", -1.0, "-1"}, + {"% .3g", -1.0, "-1"}, + {"% .3g", 1.0, " 1"}, + + // complex values + {"%+.3e", 0i, "(+0.000e+00+0.000e+00i)"}, + {"%+.3f", 0i, "(+0.000+0.000i)"}, + {"%+.3g", 0i, "(+0+0i)"}, + {"%+.3e", 1 + 2i, "(+1.000e+00+2.000e+00i)"}, + {"%+.3f", 1 + 2i, "(+1.000+2.000i)"}, + {"%+.3g", 1 + 2i, "(+1+2i)"}, + {"%.3e", 0i, "(0.000e+00+0.000e+00i)"}, + {"%.3f", 0i, "(0.000+0.000i)"}, + {"%.3g", 0i, "(0+0i)"}, + {"%.3e", 1 + 2i, "(1.000e+00+2.000e+00i)"}, + {"%.3f", 1 + 2i, "(1.000+2.000i)"}, + {"%.3g", 1 + 2i, "(1+2i)"}, + {"%.3e", -1 - 2i, "(-1.000e+00-2.000e+00i)"}, + {"%.3f", -1 - 2i, "(-1.000-2.000i)"}, + {"%.3g", -1 - 2i, "(-1-2i)"}, + {"% .3E", -1 - 2i, "(-1.000E+00-2.000E+00i)"}, + {"%+.3g", complex64(1 + 2i), "(+1+2i)"}, + {"%+.3g", complex128(1 + 2i), "(+1+2i)"}, + + // erroneous formats + {"", 2, "%!(EXTRA int=2)"}, + {"%d", "hello", "%!d(string=hello)"}, + + // old test/fmt_test.go + {"%d", 1234, "1234"}, + {"%d", -1234, "-1234"}, + {"%d", uint(1234), "1234"}, + {"%d", uint32(b32), "4294967295"}, + {"%d", uint64(b64), "18446744073709551615"}, + {"%o", 01234, "1234"}, + {"%#o", 01234, "01234"}, + {"%o", uint32(b32), "37777777777"}, + {"%o", uint64(b64), "1777777777777777777777"}, + {"%x", 0x1234abcd, "1234abcd"}, + {"%#x", 0x1234abcd, "0x1234abcd"}, + {"%x", b32 - 0x1234567, "fedcba98"}, + {"%X", 0x1234abcd, "1234ABCD"}, + {"%X", b32 - 0x1234567, "FEDCBA98"}, + {"%#X", 0, "0X0"}, + {"%x", b64, "ffffffffffffffff"}, + {"%b", 7, "111"}, + {"%b", b64, "1111111111111111111111111111111111111111111111111111111111111111"}, + {"%b", -6, "-110"}, + {"%e", 1.0, "1.000000e+00"}, + {"%e", 1234.5678e3, "1.234568e+06"}, + {"%e", 1234.5678e-8, "1.234568e-05"}, + {"%e", -7.0, "-7.000000e+00"}, + {"%e", -1e-9, "-1.000000e-09"}, + {"%f", 1234.5678e3, "1234567.800000"}, + {"%f", 1234.5678e-8, "0.000012"}, + {"%f", -7.0, "-7.000000"}, + {"%f", -1e-9, "-0.000000"}, + {"%g", 1234.5678e3, "1.2345678e+06"}, + {"%g", float32(1234.5678e3), "1.2345678e+06"}, + {"%g", 1234.5678e-8, "1.2345678e-05"}, + {"%g", -7.0, "-7"}, + {"%g", -1e-9, "-1e-09"}, + {"%g", float32(-1e-9), "-1e-09"}, + {"%E", 1.0, "1.000000E+00"}, + {"%E", 1234.5678e3, "1.234568E+06"}, + {"%E", 1234.5678e-8, "1.234568E-05"}, + {"%E", -7.0, "-7.000000E+00"}, + {"%E", -1e-9, "-1.000000E-09"}, + {"%G", 1234.5678e3, "1.2345678E+06"}, + {"%G", float32(1234.5678e3), "1.2345678E+06"}, + {"%G", 1234.5678e-8, "1.2345678E-05"}, + {"%G", -7.0, "-7"}, + {"%G", -1e-9, "-1E-09"}, + {"%G", float32(-1e-9), "-1E-09"}, + {"%c", 'x', "x"}, + {"%c", 0xe4, "ä"}, + {"%c", 0x672c, "本"}, + {"%c", '日', "日"}, + {"%20.8d", 1234, " 00001234"}, + {"%20.8d", -1234, " -00001234"}, + {"%20d", 1234, " 1234"}, + {"%-20.8d", 1234, "00001234 "}, + {"%-20.8d", -1234, "-00001234 "}, + {"%-#20.8x", 0x1234abc, "0x01234abc "}, + {"%-#20.8X", 0x1234abc, "0X01234ABC "}, + {"%-#20.8o", 01234, "00001234 "}, + {"%.20b", 7, "00000000000000000111"}, + {"%20.5s", "qwertyuiop", " qwert"}, + {"%.5s", "qwertyuiop", "qwert"}, + {"%-20.5s", "qwertyuiop", "qwert "}, + {"%20c", 'x', " x"}, + {"%-20c", 'x', "x "}, + {"%20.6e", 1.2345e3, " 1.234500e+03"}, + {"%20.6e", 1.2345e-3, " 1.234500e-03"}, + {"%20e", 1.2345e3, " 1.234500e+03"}, + {"%20e", 1.2345e-3, " 1.234500e-03"}, + {"%20.8e", 1.2345e3, " 1.23450000e+03"}, + {"%20f", 1.23456789e3, " 1234.567890"}, + {"%20f", 1.23456789e-3, " 0.001235"}, + {"%20f", 12345678901.23456789, " 12345678901.234568"}, + {"%-20f", 1.23456789e3, "1234.567890 "}, + {"%20.8f", 1.23456789e3, " 1234.56789000"}, + {"%20.8f", 1.23456789e-3, " 0.00123457"}, + {"%g", 1.23456789e3, "1234.56789"}, + {"%g", 1.23456789e-3, "0.00123456789"}, + {"%g", 1.23456789e20, "1.23456789e+20"}, + {"%20e", math.Inf(1), " +Inf"}, + {"%-20f", math.Inf(-1), "-Inf "}, + {"%20g", math.NaN(), " NaN"}, + + // arrays + {"%v", array, "[1 2 3 4 5]"}, + {"%v", iarray, "[1 hello 2.5 ]"}, + {"%v", &array, "&[1 2 3 4 5]"}, + {"%v", &iarray, "&[1 hello 2.5 ]"}, + + // complexes with %v + {"%v", 1 + 2i, "(1+2i)"}, + {"%v", complex64(1 + 2i), "(1+2i)"}, + {"%v", complex128(1 + 2i), "(1+2i)"}, + + // structs + {"%v", A{1, 2, "a", []int{1, 2}}, `{1 2 a [1 2]}`}, + {"%+v", A{1, 2, "a", []int{1, 2}}, `{i:1 j:2 s:a x:[1 2]}`}, + + // +v on structs with Stringable items + {"%+v", B{1, 2}, `{i:<1> j:2}`}, + {"%+v", C{1, B{2, 3}}, `{i:1 B:{i:<2> j:3}}`}, + + // q on Stringable items + {"%s", I(23), `<23>`}, + {"%q", I(23), `"<23>"`}, + {"%x", I(23), `3c32333e`}, + {"%d", I(23), `%!d(string=<23>)`}, + + // go syntax + {"%#v", A{1, 2, "a", []int{1, 2}}, `fmt_test.A{i:1, j:0x2, s:"a", x:[]int{1, 2}}`}, + {"%#v", &b, "(*uint8)(PTR)"}, + {"%#v", TestFmtInterface, "(func(*testing.T))(PTR)"}, + {"%#v", make(chan int), "(chan int)(PTR)"}, + {"%#v", uint64(1<<64 - 1), "0xffffffffffffffff"}, + {"%#v", 1000000000, "1000000000"}, + {"%#v", map[string]int{"a": 1, "b": 2}, `map[string] int{"a":1, "b":2}`}, + {"%#v", map[string]B{"a": {1, 2}, "b": {3, 4}}, `map[string] fmt_test.B{"a":fmt_test.B{i:1, j:2}, "b":fmt_test.B{i:3, j:4}}`}, + {"%#v", []string{"a", "b"}, `[]string{"a", "b"}`}, + + // slices with other formats + {"%#x", []int{1, 2, 15}, `[0x1 0x2 0xf]`}, + {"%x", []int{1, 2, 15}, `[1 2 f]`}, + {"%d", []int{1, 2, 15}, `[1 2 15]`}, + {"%d", []byte{1, 2, 15}, `[1 2 15]`}, + {"%q", []string{"a", "b"}, `["a" "b"]`}, + + // renamings + {"%v", renamedBool(true), "true"}, + {"%d", renamedBool(true), "%!d(fmt_test.renamedBool=true)"}, + {"%o", renamedInt(8), "10"}, + {"%d", renamedInt8(-9), "-9"}, + {"%v", renamedInt16(10), "10"}, + {"%v", renamedInt32(-11), "-11"}, + {"%X", renamedInt64(255), "FF"}, + {"%v", renamedUint(13), "13"}, + {"%o", renamedUint8(14), "16"}, + {"%X", renamedUint16(15), "F"}, + {"%d", renamedUint32(16), "16"}, + {"%X", renamedUint64(17), "11"}, + {"%o", renamedUintptr(18), "22"}, + {"%x", renamedString("thing"), "7468696e67"}, + {"%d", renamedBytes([]byte{1, 2, 15}), `[1 2 15]`}, + {"%q", renamedBytes([]byte("hello")), `"hello"`}, + {"%v", renamedFloat32(22), "22"}, + {"%v", renamedFloat64(33), "33"}, + {"%v", renamedComplex64(3 + 4i), "(3+4i)"}, + {"%v", renamedComplex128(4 - 3i), "(4-3i)"}, + + // Formatter + {"%x", F(1), ""}, + {"%x", G(2), "2"}, + {"%+v", S{F(4), G(5)}, "{f: g:5}"}, + + // GoStringer + {"%#v", G(6), "GoString(6)"}, + {"%#v", S{F(7), G(8)}, "fmt_test.S{f:, g:GoString(8)}"}, + + // %T + {"%T", (4 - 3i), "complex128"}, + {"%T", renamedComplex128(4 - 3i), "fmt_test.renamedComplex128"}, + {"%T", intVal, "int"}, + {"%6T", &intVal, " *int"}, + + // %p + {"p0=%p", new(int), "p0=PTR"}, + {"p1=%s", &pValue, "p1=String(p)"}, // String method... + {"p2=%p", &pValue, "p2=PTR"}, // ... not called with %p + + // %p on non-pointers + {"%p", make(chan int), "PTR"}, + {"%p", make(map[int]int), "PTR"}, + {"%p", make([]int, 1), "PTR"}, + {"%p", 27, "%!p(int=27)"}, // not a pointer at all + + // erroneous things + {"%s %", "hello", "hello %!(NOVERB)"}, + {"%s %.2", "hello", "hello %!(NOVERB)"}, + {"%d", "hello", "%!d(string=hello)"}, + {"no args", "hello", "no args%!(EXTRA string=hello)"}, + {"%s", nil, "%!s()"}, + {"%T", nil, ""}, + {"%-1", 100, "%!(NOVERB)%!(EXTRA int=100)"}, +} + +func TestSprintf(t *testing.T) { + for _, tt := range fmttests { + s := Sprintf(tt.fmt, tt.val) + if i := strings.Index(s, "0x"); i >= 0 && strings.Contains(tt.out, "PTR") { + j := i + 2 + for ; j < len(s); j++ { + c := s[j] + if (c < '0' || c > '9') && (c < 'a' || c > 'f') && (c < 'A' || c > 'F') { + break + } + } + s = s[0:i] + "PTR" + s[j:] + } + if s != tt.out { + if _, ok := tt.val.(string); ok { + // Don't requote the already-quoted strings. + // It's too confusing to read the errors. + t.Errorf("Sprintf(%q, %q) = <%s> want <%s>", tt.fmt, tt.val, s, tt.out) + } else { + t.Errorf("Sprintf(%q, %v) = %q want %q", tt.fmt, tt.val, s, tt.out) + } + } + } +} + +func BenchmarkSprintfEmpty(b *testing.B) { + for i := 0; i < b.N; i++ { + Sprintf("") + } +} + +func BenchmarkSprintfString(b *testing.B) { + for i := 0; i < b.N; i++ { + Sprintf("%s", "hello") + } +} + +func BenchmarkSprintfInt(b *testing.B) { + for i := 0; i < b.N; i++ { + Sprintf("%d", 5) + } +} + +func BenchmarkSprintfIntInt(b *testing.B) { + for i := 0; i < b.N; i++ { + Sprintf("%d %d", 5, 6) + } +} + +func BenchmarkSprintfPrefixedInt(b *testing.B) { + for i := 0; i < b.N; i++ { + Sprintf("This is some meaningless prefix text that needs to be scanned %d", 6) + } +} + +func TestCountMallocs(t *testing.T) { + mallocs := 0 - runtime.MemStats.Mallocs + for i := 0; i < 100; i++ { + Sprintf("") + } + mallocs += runtime.MemStats.Mallocs + Printf("mallocs per Sprintf(\"\"): %d\n", mallocs/100) + mallocs = 0 - runtime.MemStats.Mallocs + for i := 0; i < 100; i++ { + Sprintf("xxx") + } + mallocs += runtime.MemStats.Mallocs + Printf("mallocs per Sprintf(\"xxx\"): %d\n", mallocs/100) + mallocs = 0 - runtime.MemStats.Mallocs + for i := 0; i < 100; i++ { + Sprintf("%x", i) + } + mallocs += runtime.MemStats.Mallocs + Printf("mallocs per Sprintf(\"%%x\"): %d\n", mallocs/100) + mallocs = 0 - runtime.MemStats.Mallocs + for i := 0; i < 100; i++ { + Sprintf("%x %x", i, i) + } + mallocs += runtime.MemStats.Mallocs + Printf("mallocs per Sprintf(\"%%x %%x\"): %d\n", mallocs/100) +} + +type flagPrinter struct{} + +func (*flagPrinter) Format(f State, c int) { + s := "%" + for i := 0; i < 128; i++ { + if f.Flag(i) { + s += string(i) + } + } + if w, ok := f.Width(); ok { + s += Sprintf("%d", w) + } + if p, ok := f.Precision(); ok { + s += Sprintf(".%d", p) + } + s += string(c) + io.WriteString(f, "["+s+"]") +} + +var flagtests = []struct { + in string + out string +}{ + {"%a", "[%a]"}, + {"%-a", "[%-a]"}, + {"%+a", "[%+a]"}, + {"%#a", "[%#a]"}, + {"% a", "[% a]"}, + {"%0a", "[%0a]"}, + {"%1.2a", "[%1.2a]"}, + {"%-1.2a", "[%-1.2a]"}, + {"%+1.2a", "[%+1.2a]"}, + {"%-+1.2a", "[%+-1.2a]"}, + {"%-+1.2abc", "[%+-1.2a]bc"}, + {"%-1.2abc", "[%-1.2a]bc"}, +} + +func TestFlagParser(t *testing.T) { + var flagprinter flagPrinter + for _, tt := range flagtests { + s := Sprintf(tt.in, &flagprinter) + if s != tt.out { + t.Errorf("Sprintf(%q, &flagprinter) => %q, want %q", tt.in, s, tt.out) + } + } +} + +func TestStructPrinter(t *testing.T) { + var s struct { + a string + b string + c int + } + s.a = "abc" + s.b = "def" + s.c = 123 + var tests = []struct { + fmt string + out string + }{ + {"%v", "{abc def 123}"}, + {"%+v", "{a:abc b:def c:123}"}, + } + for _, tt := range tests { + out := Sprintf(tt.fmt, s) + if out != tt.out { + t.Errorf("Sprintf(%q, &s) = %q, want %q", tt.fmt, out, tt.out) + } + } +} + +// Check map printing using substrings so we don't depend on the print order. +func presentInMap(s string, a []string, t *testing.T) { + for i := 0; i < len(a); i++ { + loc := strings.Index(s, a[i]) + if loc < 0 { + t.Errorf("map print: expected to find %q in %q", a[i], s) + } + // make sure the match ends here + loc += len(a[i]) + if loc >= len(s) || (s[loc] != ' ' && s[loc] != ']') { + t.Errorf("map print: %q not properly terminated in %q", a[i], s) + } + } +} + +func TestMapPrinter(t *testing.T) { + m0 := make(map[int]string) + s := Sprint(m0) + if s != "map[]" { + t.Errorf("empty map printed as %q not %q", s, "map[]") + } + m1 := map[int]string{1: "one", 2: "two", 3: "three"} + a := []string{"1:one", "2:two", "3:three"} + presentInMap(Sprintf("%v", m1), a, t) + presentInMap(Sprint(m1), a, t) +} + +func TestEmptyMap(t *testing.T) { + const emptyMapStr = "map[]" + var m map[string]int + s := Sprint(m) + if s != emptyMapStr { + t.Errorf("nil map printed as %q not %q", s, emptyMapStr) + } + m = make(map[string]int) + s = Sprint(m) + if s != emptyMapStr { + t.Errorf("empty map printed as %q not %q", s, emptyMapStr) + } +} + +// Check that Sprint (and hence Print, Fprint) puts spaces in the right places, +// that is, between arg pairs in which neither is a string. +func TestBlank(t *testing.T) { + got := Sprint("<", 1, ">:", 1, 2, 3, "!") + expect := "<1>:1 2 3!" + if got != expect { + t.Errorf("got %q expected %q", got, expect) + } +} + +// Check that Sprintln (and hence Println, Fprintln) puts spaces in the right places, +// that is, between all arg pairs. +func TestBlankln(t *testing.T) { + got := Sprintln("<", 1, ">:", 1, 2, 3, "!") + expect := "< 1 >: 1 2 3 !\n" + if got != expect { + t.Errorf("got %q expected %q", got, expect) + } +} + + +// Check Formatter with Sprint, Sprintln, Sprintf +func TestFormatterPrintln(t *testing.T) { + f := F(1) + expect := "\n" + s := Sprint(f, "\n") + if s != expect { + t.Errorf("Sprint wrong with Formatter: expected %q got %q", expect, s) + } + s = Sprintln(f) + if s != expect { + t.Errorf("Sprintln wrong with Formatter: expected %q got %q", expect, s) + } + s = Sprintf("%v\n", f) + if s != expect { + t.Errorf("Sprintf wrong with Formatter: expected %q got %q", expect, s) + } +} + +func args(a ...interface{}) []interface{} { return a } + +var startests = []struct { + fmt string + in []interface{} + out string +}{ + {"%*d", args(4, 42), " 42"}, + {"%.*d", args(4, 42), "0042"}, + {"%*.*d", args(8, 4, 42), " 0042"}, + {"%0*d", args(4, 42), "0042"}, + {"%-*d", args(4, 42), "42 "}, + + // erroneous + {"%*d", args(nil, 42), "%!(BADWIDTH)42"}, + {"%.*d", args(nil, 42), "%!(BADPREC)42"}, + {"%*d", args(5, "foo"), "%!d(string= foo)"}, + {"%*% %d", args(20, 5), "% 5"}, + {"%*", args(4), "%!(NOVERB)"}, + {"%*d", args(int32(4), 42), "%!(BADWIDTH)42"}, +} + +func TestWidthAndPrecision(t *testing.T) { + for _, tt := range startests { + s := Sprintf(tt.fmt, tt.in...) + if s != tt.out { + t.Errorf("%q: got %q expected %q", tt.fmt, s, tt.out) + } + } +} diff --git a/libgo/go/fmt/format.go b/libgo/go/fmt/format.go new file mode 100644 index 000000000..86057bf69 --- /dev/null +++ b/libgo/go/fmt/format.go @@ -0,0 +1,398 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fmt + +import ( + "bytes" + "strconv" + "utf8" +) + +const ( + nByte = 64 + + ldigits = "0123456789abcdef" + udigits = "0123456789ABCDEF" +) + +const ( + signed = true + unsigned = false +) + +var padZeroBytes = make([]byte, nByte) +var padSpaceBytes = make([]byte, nByte) + +var newline = []byte{'\n'} + +func init() { + for i := 0; i < nByte; i++ { + padZeroBytes[i] = '0' + padSpaceBytes[i] = ' ' + } +} + +// A fmt is the raw formatter used by Printf etc. +// It prints into a bytes.Buffer that must be set up externally. +type fmt struct { + intbuf [nByte]byte + buf *bytes.Buffer + // width, precision + wid int + prec int + // flags + widPresent bool + precPresent bool + minus bool + plus bool + sharp bool + space bool + unicode bool + zero bool +} + +func (f *fmt) clearflags() { + f.wid = 0 + f.widPresent = false + f.prec = 0 + f.precPresent = false + f.minus = false + f.plus = false + f.sharp = false + f.space = false + f.unicode = false + f.zero = false +} + +func (f *fmt) init(buf *bytes.Buffer) { + f.buf = buf + f.clearflags() +} + +// Compute left and right padding widths (only one will be non-zero). +func (f *fmt) computePadding(width int) (padding []byte, leftWidth, rightWidth int) { + left := !f.minus + w := f.wid + if w < 0 { + left = false + w = -w + } + w -= width + if w > 0 { + if left && f.zero { + return padZeroBytes, w, 0 + } + if left { + return padSpaceBytes, w, 0 + } else { + // can't be zero padding on the right + return padSpaceBytes, 0, w + } + } + return +} + +// Generate n bytes of padding. +func (f *fmt) writePadding(n int, padding []byte) { + for n > 0 { + m := n + if m > nByte { + m = nByte + } + f.buf.Write(padding[0:m]) + n -= m + } +} + +// Append b to f.buf, padded on left (w > 0) or right (w < 0 or f.minus) +// clear flags aftewards. +func (f *fmt) pad(b []byte) { + var padding []byte + var left, right int + if f.widPresent && f.wid != 0 { + padding, left, right = f.computePadding(len(b)) + } + if left > 0 { + f.writePadding(left, padding) + } + f.buf.Write(b) + if right > 0 { + f.writePadding(right, padding) + } +} + +// append s to buf, padded on left (w > 0) or right (w < 0 or f.minus). +// clear flags aftewards. +func (f *fmt) padString(s string) { + var padding []byte + var left, right int + if f.widPresent && f.wid != 0 { + padding, left, right = f.computePadding(utf8.RuneCountInString(s)) + } + if left > 0 { + f.writePadding(left, padding) + } + f.buf.WriteString(s) + if right > 0 { + f.writePadding(right, padding) + } +} + +func putint(buf []byte, base, val uint64, digits string) int { + i := len(buf) - 1 + for val >= base { + buf[i] = digits[val%base] + i-- + val /= base + } + buf[i] = digits[val] + return i - 1 +} + +// fmt_boolean formats a boolean. +func (f *fmt) fmt_boolean(v bool) { + if v { + f.padString("true") + } else { + f.padString("false") + } +} + +// integer; interprets prec but not wid. Once formatted, result is sent to pad() +// and then flags are cleared. +func (f *fmt) integer(a int64, base uint64, signedness bool, digits string) { + var buf []byte = f.intbuf[0:] + negative := signedness == signed && a < 0 + if negative { + a = -a + } + + // two ways to ask for extra leading zero digits: %.3d or %03d. + // apparently the first cancels the second. + prec := 0 + if f.precPresent { + prec = f.prec + f.zero = false + } else if f.zero && f.widPresent && !f.minus && f.wid > 0 { + prec = f.wid + if negative || f.plus || f.space { + prec-- // leave room for sign + } + } + + // format a into buf, ending at buf[i]. (printing is easier right-to-left.) + // a is made into unsigned ua. we could make things + // marginally faster by splitting the 32-bit case out into a separate + // block but it's not worth the duplication, so ua has 64 bits. + i := len(f.intbuf) + ua := uint64(a) + for ua >= base { + i-- + buf[i] = digits[ua%base] + ua /= base + } + i-- + buf[i] = digits[ua] + for i > 0 && prec > nByte-i { + i-- + buf[i] = '0' + } + + // Various prefixes: 0x, -, etc. + if f.sharp { + switch base { + case 8: + if buf[i] != '0' { + i-- + buf[i] = '0' + } + case 16: + i-- + buf[i] = 'x' + digits[10] - 'a' + i-- + buf[i] = '0' + } + } + if f.unicode { + i-- + buf[i] = '+' + i-- + buf[i] = 'U' + } + + if negative { + i-- + buf[i] = '-' + } else if f.plus { + i-- + buf[i] = '+' + } else if f.space { + i-- + buf[i] = ' ' + } + f.pad(buf[i:]) +} + +// fmt_s formats a string. +func (f *fmt) fmt_s(s string) { + if f.precPresent { + if f.prec < len(s) { + s = s[0:f.prec] + } + } + f.padString(s) +} + +// fmt_sx formats a string as a hexadecimal encoding of its bytes. +func (f *fmt) fmt_sx(s string) { + t := "" + for i := 0; i < len(s); i++ { + if i > 0 && f.space { + t += " " + } + v := s[i] + t += string(ldigits[v>>4]) + t += string(ldigits[v&0xF]) + } + f.padString(t) +} + +// fmt_sX formats a string as an uppercase hexadecimal encoding of its bytes. +func (f *fmt) fmt_sX(s string) { + t := "" + for i := 0; i < len(s); i++ { + if i > 0 && f.space { + t += " " + } + v := s[i] + t += string(udigits[v>>4]) + t += string(udigits[v&0xF]) + } + f.padString(t) +} + +// fmt_q formats a string as a double-quoted, escaped Go string constant. +func (f *fmt) fmt_q(s string) { + var quoted string + if f.sharp && strconv.CanBackquote(s) { + quoted = "`" + s + "`" + } else { + quoted = strconv.Quote(s) + } + f.padString(quoted) +} + +// floating-point + +func doPrec(f *fmt, def int) int { + if f.precPresent { + return f.prec + } + return def +} + +// Add a plus sign or space to the floating-point string representation if missing and required. +func (f *fmt) plusSpace(s string) { + if s[0] != '-' { + if f.plus { + s = "+" + s + } else if f.space { + s = " " + s + } + } + f.padString(s) +} + +// fmt_e64 formats a float64 in the form -1.23e+12. +func (f *fmt) fmt_e64(v float64) { f.plusSpace(strconv.Ftoa64(v, 'e', doPrec(f, 6))) } + +// fmt_E64 formats a float64 in the form -1.23E+12. +func (f *fmt) fmt_E64(v float64) { f.plusSpace(strconv.Ftoa64(v, 'E', doPrec(f, 6))) } + +// fmt_f64 formats a float64 in the form -1.23. +func (f *fmt) fmt_f64(v float64) { f.plusSpace(strconv.Ftoa64(v, 'f', doPrec(f, 6))) } + +// fmt_g64 formats a float64 in the 'f' or 'e' form according to size. +func (f *fmt) fmt_g64(v float64) { f.plusSpace(strconv.Ftoa64(v, 'g', doPrec(f, -1))) } + +// fmt_g64 formats a float64 in the 'f' or 'E' form according to size. +func (f *fmt) fmt_G64(v float64) { f.plusSpace(strconv.Ftoa64(v, 'G', doPrec(f, -1))) } + +// fmt_fb64 formats a float64 in the form -123p3 (exponent is power of 2). +func (f *fmt) fmt_fb64(v float64) { f.plusSpace(strconv.Ftoa64(v, 'b', 0)) } + +// float32 +// cannot defer to float64 versions +// because it will get rounding wrong in corner cases. + +// fmt_e32 formats a float32 in the form -1.23e+12. +func (f *fmt) fmt_e32(v float32) { f.plusSpace(strconv.Ftoa32(v, 'e', doPrec(f, 6))) } + +// fmt_E32 formats a float32 in the form -1.23E+12. +func (f *fmt) fmt_E32(v float32) { f.plusSpace(strconv.Ftoa32(v, 'E', doPrec(f, 6))) } + +// fmt_f32 formats a float32 in the form -1.23. +func (f *fmt) fmt_f32(v float32) { f.plusSpace(strconv.Ftoa32(v, 'f', doPrec(f, 6))) } + +// fmt_g32 formats a float32 in the 'f' or 'e' form according to size. +func (f *fmt) fmt_g32(v float32) { f.plusSpace(strconv.Ftoa32(v, 'g', doPrec(f, -1))) } + +// fmt_G32 formats a float32 in the 'f' or 'E' form according to size. +func (f *fmt) fmt_G32(v float32) { f.plusSpace(strconv.Ftoa32(v, 'G', doPrec(f, -1))) } + +// fmt_fb32 formats a float32 in the form -123p3 (exponent is power of 2). +func (f *fmt) fmt_fb32(v float32) { f.padString(strconv.Ftoa32(v, 'b', 0)) } + +// fmt_c64 formats a complex64 according to the verb. +func (f *fmt) fmt_c64(v complex64, verb int) { + f.buf.WriteByte('(') + r := real(v) + for i := 0; ; i++ { + switch verb { + case 'e': + f.fmt_e32(r) + case 'E': + f.fmt_E32(r) + case 'f': + f.fmt_f32(r) + case 'g': + f.fmt_g32(r) + case 'G': + f.fmt_G32(r) + } + if i != 0 { + break + } + f.plus = true + r = imag(v) + } + f.buf.Write(irparenBytes) +} + +// fmt_c128 formats a complex128 according to the verb. +func (f *fmt) fmt_c128(v complex128, verb int) { + f.buf.WriteByte('(') + r := real(v) + for i := 0; ; i++ { + switch verb { + case 'e': + f.fmt_e64(r) + case 'E': + f.fmt_E64(r) + case 'f': + f.fmt_f64(r) + case 'g': + f.fmt_g64(r) + case 'G': + f.fmt_G64(r) + } + if i != 0 { + break + } + f.plus = true + r = imag(v) + } + f.buf.Write(irparenBytes) +} diff --git a/libgo/go/fmt/print.go b/libgo/go/fmt/print.go new file mode 100644 index 000000000..96029a878 --- /dev/null +++ b/libgo/go/fmt/print.go @@ -0,0 +1,921 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fmt + +import ( + "bytes" + "io" + "os" + "reflect" + "utf8" +) + +// Some constants in the form of bytes, to avoid string overhead. +// Needlessly fastidious, I suppose. +var ( + commaSpaceBytes = []byte(", ") + nilAngleBytes = []byte("") + nilParenBytes = []byte("(nil)") + nilBytes = []byte("nil") + mapBytes = []byte("map[") + missingBytes = []byte("(MISSING)") + extraBytes = []byte("%!(EXTRA ") + irparenBytes = []byte("i)") + bytesBytes = []byte("[]byte{") + widthBytes = []byte("%!(BADWIDTH)") + precBytes = []byte("%!(BADPREC)") + noVerbBytes = []byte("%!(NOVERB)") +) + +// State represents the printer state passed to custom formatters. +// It provides access to the io.Writer interface plus information about +// the flags and options for the operand's format specifier. +type State interface { + // Write is the function to call to emit formatted output to be printed. + Write(b []byte) (ret int, err os.Error) + // Width returns the value of the width option and whether it has been set. + Width() (wid int, ok bool) + // Precision returns the value of the precision option and whether it has been set. + Precision() (prec int, ok bool) + + // Flag returns whether the flag c, a character, has been set. + Flag(int) bool +} + +// Formatter is the interface implemented by values with a custom formatter. +// The implementation of Format may call Sprintf or Fprintf(f) etc. +// to generate its output. +type Formatter interface { + Format(f State, c int) +} + +// Stringer is implemented by any value that has a String method(), +// which defines the ``native'' format for that value. +// The String method is used to print values passed as an operand +// to a %s or %v format or to an unformatted printer such as Print. +type Stringer interface { + String() string +} + +// GoStringer is implemented by any value that has a GoString() method, +// which defines the Go syntax for that value. +// The GoString method is used to print values passed as an operand +// to a %#v format. +type GoStringer interface { + GoString() string +} + +type pp struct { + n int + buf bytes.Buffer + runeBuf [utf8.UTFMax]byte + fmt fmt +} + +// A leaky bucket of reusable pp structures. +var ppFree = make(chan *pp, 100) + +// Allocate a new pp struct. Probably can grab the previous one from ppFree. +func newPrinter() *pp { + p, ok := <-ppFree + if !ok { + p = new(pp) + } + p.fmt.init(&p.buf) + return p +} + +// Save used pp structs in ppFree; avoids an allocation per invocation. +func (p *pp) free() { + // Don't hold on to pp structs with large buffers. + if cap(p.buf.Bytes()) > 1024 { + return + } + p.buf.Reset() + _ = ppFree <- p +} + +func (p *pp) Width() (wid int, ok bool) { return p.fmt.wid, p.fmt.widPresent } + +func (p *pp) Precision() (prec int, ok bool) { return p.fmt.prec, p.fmt.precPresent } + +func (p *pp) Flag(b int) bool { + switch b { + case '-': + return p.fmt.minus + case '+': + return p.fmt.plus + case '#': + return p.fmt.sharp + case ' ': + return p.fmt.space + case '0': + return p.fmt.zero + } + return false +} + +func (p *pp) add(c int) { + p.buf.WriteRune(c) +} + +// Implement Write so we can call Fprintf on a pp (through State), for +// recursive use in custom verbs. +func (p *pp) Write(b []byte) (ret int, err os.Error) { + return p.buf.Write(b) +} + +// These routines end in 'f' and take a format string. + +// Fprintf formats according to a format specifier and writes to w. +// It returns the number of bytes written and any write error encountered. +func Fprintf(w io.Writer, format string, a ...interface{}) (n int, error os.Error) { + p := newPrinter() + p.doPrintf(format, a) + n64, error := p.buf.WriteTo(w) + p.free() + return int(n64), error +} + +// Printf formats according to a format specifier and writes to standard output. +// It returns the number of bytes written and any write error encountered. +func Printf(format string, a ...interface{}) (n int, errno os.Error) { + n, errno = Fprintf(os.Stdout, format, a...) + return n, errno +} + +// Sprintf formats according to a format specifier and returns the resulting string. +func Sprintf(format string, a ...interface{}) string { + p := newPrinter() + p.doPrintf(format, a) + s := p.buf.String() + p.free() + return s +} + +// Errorf formats according to a format specifier and returns the string +// converted to an os.ErrorString, which satisfies the os.Error interface. +func Errorf(format string, a ...interface{}) os.Error { + return os.ErrorString(Sprintf(format, a...)) +} + +// These routines do not take a format string + +// Fprint formats using the default formats for its operands and writes to w. +// Spaces are added between operands when neither is a string. +// It returns the number of bytes written and any write error encountered. +func Fprint(w io.Writer, a ...interface{}) (n int, error os.Error) { + p := newPrinter() + p.doPrint(a, false, false) + n64, error := p.buf.WriteTo(w) + p.free() + return int(n64), error +} + +// Print formats using the default formats for its operands and writes to standard output. +// Spaces are added between operands when neither is a string. +// It returns the number of bytes written and any write error encountered. +func Print(a ...interface{}) (n int, errno os.Error) { + n, errno = Fprint(os.Stdout, a...) + return n, errno +} + +// Sprint formats using the default formats for its operands and returns the resulting string. +// Spaces are added between operands when neither is a string. +func Sprint(a ...interface{}) string { + p := newPrinter() + p.doPrint(a, false, false) + s := p.buf.String() + p.free() + return s +} + +// These routines end in 'ln', do not take a format string, +// always add spaces between operands, and add a newline +// after the last operand. + +// Fprintln formats using the default formats for its operands and writes to w. +// Spaces are always added between operands and a newline is appended. +// It returns the number of bytes written and any write error encountered. +func Fprintln(w io.Writer, a ...interface{}) (n int, error os.Error) { + p := newPrinter() + p.doPrint(a, true, true) + n64, error := p.buf.WriteTo(w) + p.free() + return int(n64), error +} + +// Println formats using the default formats for its operands and writes to standard output. +// Spaces are always added between operands and a newline is appended. +// It returns the number of bytes written and any write error encountered. +func Println(a ...interface{}) (n int, errno os.Error) { + n, errno = Fprintln(os.Stdout, a...) + return n, errno +} + +// Sprintln formats using the default formats for its operands and returns the resulting string. +// Spaces are always added between operands and a newline is appended. +func Sprintln(a ...interface{}) string { + p := newPrinter() + p.doPrint(a, true, true) + s := p.buf.String() + p.free() + return s +} + + +// Get the i'th arg of the struct value. +// If the arg itself is an interface, return a value for +// the thing inside the interface, not the interface itself. +func getField(v *reflect.StructValue, i int) reflect.Value { + val := v.Field(i) + if i, ok := val.(*reflect.InterfaceValue); ok { + if inter := i.Interface(); inter != nil { + return reflect.NewValue(inter) + } + } + return val +} + +// Convert ASCII to integer. n is 0 (and got is false) if no number present. +func parsenum(s string, start, end int) (num int, isnum bool, newi int) { + if start >= end { + return 0, false, end + } + for newi = start; newi < end && '0' <= s[newi] && s[newi] <= '9'; newi++ { + num = num*10 + int(s[newi]-'0') + isnum = true + } + return +} + +// Reflection values like reflect.FuncValue implement this method. We use it for %p. +type uintptrGetter interface { + Get() uintptr +} + +func (p *pp) unknownType(v interface{}) { + if v == nil { + p.buf.Write(nilAngleBytes) + return + } + p.buf.WriteByte('?') + p.buf.WriteString(reflect.Typeof(v).String()) + p.buf.WriteByte('?') +} + +func (p *pp) badVerb(verb int, val interface{}) { + p.add('%') + p.add('!') + p.add(verb) + p.add('(') + if val == nil { + p.buf.Write(nilAngleBytes) + } else { + p.buf.WriteString(reflect.Typeof(val).String()) + p.add('=') + p.printField(val, 'v', false, false, 0) + } + p.add(')') +} + +func (p *pp) fmtBool(v bool, verb int, value interface{}) { + switch verb { + case 't', 'v': + p.fmt.fmt_boolean(v) + default: + p.badVerb(verb, value) + } +} + +// fmtC formats a rune for the 'c' format. +func (p *pp) fmtC(c int64) { + rune := int(c) // Check for overflow. + if int64(rune) != c { + rune = utf8.RuneError + } + w := utf8.EncodeRune(p.runeBuf[0:utf8.UTFMax], rune) + p.fmt.pad(p.runeBuf[0:w]) +} + +func (p *pp) fmtInt64(v int64, verb int, value interface{}) { + switch verb { + case 'b': + p.fmt.integer(v, 2, signed, ldigits) + case 'c': + p.fmtC(v) + case 'd', 'v': + p.fmt.integer(v, 10, signed, ldigits) + case 'o': + p.fmt.integer(v, 8, signed, ldigits) + case 'x': + p.fmt.integer(v, 16, signed, ldigits) + case 'U': + p.fmtUnicode(v) + case 'X': + p.fmt.integer(v, 16, signed, udigits) + default: + p.badVerb(verb, value) + } +} + +// fmt0x64 formats a uint64 in hexadecimal and prefixes it with 0x by +// temporarily turning on the sharp flag. +func (p *pp) fmt0x64(v uint64) { + sharp := p.fmt.sharp + p.fmt.sharp = true // turn on 0x + p.fmt.integer(int64(v), 16, unsigned, ldigits) + p.fmt.sharp = sharp +} + +// fmtUnicode formats a uint64 in U+1234 form by +// temporarily turning on the unicode flag and tweaking the precision. +func (p *pp) fmtUnicode(v int64) { + precPresent := p.fmt.precPresent + prec := p.fmt.prec + if !precPresent { + // If prec is already set, leave it alone; otherwise 4 is minimum. + p.fmt.prec = 4 + p.fmt.precPresent = true + } + p.fmt.unicode = true // turn on U+ + p.fmt.integer(int64(v), 16, unsigned, udigits) + p.fmt.unicode = false + p.fmt.prec = prec + p.fmt.precPresent = precPresent +} + +func (p *pp) fmtUint64(v uint64, verb int, goSyntax bool, value interface{}) { + switch verb { + case 'b': + p.fmt.integer(int64(v), 2, unsigned, ldigits) + case 'c': + p.fmtC(int64(v)) + case 'd': + p.fmt.integer(int64(v), 10, unsigned, ldigits) + case 'v': + if goSyntax { + p.fmt0x64(v) + } else { + p.fmt.integer(int64(v), 10, unsigned, ldigits) + } + case 'o': + p.fmt.integer(int64(v), 8, unsigned, ldigits) + case 'x': + p.fmt.integer(int64(v), 16, unsigned, ldigits) + case 'X': + p.fmt.integer(int64(v), 16, unsigned, udigits) + default: + p.badVerb(verb, value) + } +} + +func (p *pp) fmtFloat32(v float32, verb int, value interface{}) { + switch verb { + case 'b': + p.fmt.fmt_fb32(v) + case 'e': + p.fmt.fmt_e32(v) + case 'E': + p.fmt.fmt_E32(v) + case 'f': + p.fmt.fmt_f32(v) + case 'g', 'v': + p.fmt.fmt_g32(v) + case 'G': + p.fmt.fmt_G32(v) + default: + p.badVerb(verb, value) + } +} + +func (p *pp) fmtFloat64(v float64, verb int, value interface{}) { + switch verb { + case 'b': + p.fmt.fmt_fb64(v) + case 'e': + p.fmt.fmt_e64(v) + case 'E': + p.fmt.fmt_E64(v) + case 'f': + p.fmt.fmt_f64(v) + case 'g', 'v': + p.fmt.fmt_g64(v) + case 'G': + p.fmt.fmt_G64(v) + default: + p.badVerb(verb, value) + } +} + +func (p *pp) fmtComplex64(v complex64, verb int, value interface{}) { + switch verb { + case 'e', 'E', 'f', 'F', 'g', 'G': + p.fmt.fmt_c64(v, verb) + case 'v': + p.fmt.fmt_c64(v, 'g') + default: + p.badVerb(verb, value) + } +} + +func (p *pp) fmtComplex128(v complex128, verb int, value interface{}) { + switch verb { + case 'e', 'E', 'f', 'F', 'g', 'G': + p.fmt.fmt_c128(v, verb) + case 'v': + p.fmt.fmt_c128(v, 'g') + default: + p.badVerb(verb, value) + } +} + +func (p *pp) fmtString(v string, verb int, goSyntax bool, value interface{}) { + switch verb { + case 'v': + if goSyntax { + p.fmt.fmt_q(v) + } else { + p.fmt.fmt_s(v) + } + case 's': + p.fmt.fmt_s(v) + case 'x': + p.fmt.fmt_sx(v) + case 'X': + p.fmt.fmt_sX(v) + case 'q': + p.fmt.fmt_q(v) + default: + p.badVerb(verb, value) + } +} + +func (p *pp) fmtBytes(v []byte, verb int, goSyntax bool, depth int, value interface{}) { + if verb == 'v' || verb == 'd' { + if goSyntax { + p.buf.Write(bytesBytes) + } else { + p.buf.WriteByte('[') + } + for i, c := range v { + if i > 0 { + if goSyntax { + p.buf.Write(commaSpaceBytes) + } else { + p.buf.WriteByte(' ') + } + } + p.printField(c, 'v', p.fmt.plus, goSyntax, depth+1) + } + if goSyntax { + p.buf.WriteByte('}') + } else { + p.buf.WriteByte(']') + } + return + } + s := string(v) + switch verb { + case 's': + p.fmt.fmt_s(s) + case 'x': + p.fmt.fmt_sx(s) + case 'X': + p.fmt.fmt_sX(s) + case 'q': + p.fmt.fmt_q(s) + default: + p.badVerb(verb, value) + } +} + +func (p *pp) fmtPointer(field interface{}, value reflect.Value, verb int, goSyntax bool) { + v, ok := value.(uintptrGetter) + if !ok { // reflect.PtrValue is a uintptrGetter, so failure means it's not a pointer at all. + p.badVerb(verb, field) + return + } + u := v.Get() + if goSyntax { + p.add('(') + p.buf.WriteString(reflect.Typeof(field).String()) + p.add(')') + p.add('(') + if u == 0 { + p.buf.Write(nilBytes) + } else { + p.fmt0x64(uint64(v.Get())) + } + p.add(')') + } else { + p.fmt0x64(uint64(u)) + } +} + +var ( + intBits = reflect.Typeof(0).Bits() + floatBits = reflect.Typeof(0.0).Bits() + complexBits = reflect.Typeof(1i).Bits() + uintptrBits = reflect.Typeof(uintptr(0)).Bits() +) + +func (p *pp) printField(field interface{}, verb int, plus, goSyntax bool, depth int) (wasString bool) { + if field == nil { + if verb == 'T' || verb == 'v' { + p.buf.Write(nilAngleBytes) + } else { + p.badVerb(verb, field) + } + return false + } + + // Special processing considerations. + // %T (the value's type) and %p (its address) are special; we always do them first. + switch verb { + case 'T': + p.printField(reflect.Typeof(field).String(), 's', false, false, 0) + return false + case 'p': + p.fmtPointer(field, reflect.NewValue(field), verb, goSyntax) + return false + } + // Is it a Formatter? + if formatter, ok := field.(Formatter); ok { + formatter.Format(p, verb) + return false // this value is not a string + + } + // Must not touch flags before Formatter looks at them. + if plus { + p.fmt.plus = false + } + // If we're doing Go syntax and the field knows how to supply it, take care of it now. + if goSyntax { + p.fmt.sharp = false + if stringer, ok := field.(GoStringer); ok { + // Print the result of GoString unadorned. + p.fmtString(stringer.GoString(), 's', false, field) + return false // this value is not a string + } + } else { + // Is it a Stringer? + if stringer, ok := field.(Stringer); ok { + p.printField(stringer.String(), verb, plus, false, depth) + return false // this value is not a string + } + } + + // Some types can be done without reflection. + switch f := field.(type) { + case bool: + p.fmtBool(f, verb, field) + return false + case float32: + p.fmtFloat32(f, verb, field) + return false + case float64: + p.fmtFloat64(f, verb, field) + return false + case complex64: + p.fmtComplex64(complex64(f), verb, field) + return false + case complex128: + p.fmtComplex128(f, verb, field) + return false + case int: + p.fmtInt64(int64(f), verb, field) + return false + case int8: + p.fmtInt64(int64(f), verb, field) + return false + case int16: + p.fmtInt64(int64(f), verb, field) + return false + case int32: + p.fmtInt64(int64(f), verb, field) + return false + case int64: + p.fmtInt64(f, verb, field) + return false + case uint: + p.fmtUint64(uint64(f), verb, goSyntax, field) + return false + case uint8: + p.fmtUint64(uint64(f), verb, goSyntax, field) + return false + case uint16: + p.fmtUint64(uint64(f), verb, goSyntax, field) + return false + case uint32: + p.fmtUint64(uint64(f), verb, goSyntax, field) + return false + case uint64: + p.fmtUint64(f, verb, goSyntax, field) + return false + case uintptr: + p.fmtUint64(uint64(f), verb, goSyntax, field) + return false + case string: + p.fmtString(f, verb, goSyntax, field) + return verb == 's' || verb == 'v' + case []byte: + p.fmtBytes(f, verb, goSyntax, depth, field) + return verb == 's' + } + + // Need to use reflection + value := reflect.NewValue(field) + +BigSwitch: + switch f := value.(type) { + case *reflect.BoolValue: + p.fmtBool(f.Get(), verb, field) + case *reflect.IntValue: + p.fmtInt64(f.Get(), verb, field) + case *reflect.UintValue: + p.fmtUint64(uint64(f.Get()), verb, goSyntax, field) + case *reflect.FloatValue: + if f.Type().Size() == 4 { + p.fmtFloat32(float32(f.Get()), verb, field) + } else { + p.fmtFloat64(float64(f.Get()), verb, field) + } + case *reflect.ComplexValue: + if f.Type().Size() == 8 { + p.fmtComplex64(complex64(f.Get()), verb, field) + } else { + p.fmtComplex128(complex128(f.Get()), verb, field) + } + case *reflect.StringValue: + p.fmtString(f.Get(), verb, goSyntax, field) + case *reflect.MapValue: + if goSyntax { + p.buf.WriteString(f.Type().String()) + p.buf.WriteByte('{') + } else { + p.buf.Write(mapBytes) + } + keys := f.Keys() + for i, key := range keys { + if i > 0 { + if goSyntax { + p.buf.Write(commaSpaceBytes) + } else { + p.buf.WriteByte(' ') + } + } + p.printField(key.Interface(), verb, plus, goSyntax, depth+1) + p.buf.WriteByte(':') + p.printField(f.Elem(key).Interface(), verb, plus, goSyntax, depth+1) + } + if goSyntax { + p.buf.WriteByte('}') + } else { + p.buf.WriteByte(']') + } + case *reflect.StructValue: + if goSyntax { + p.buf.WriteString(reflect.Typeof(field).String()) + } + p.add('{') + v := f + t := v.Type().(*reflect.StructType) + for i := 0; i < v.NumField(); i++ { + if i > 0 { + if goSyntax { + p.buf.Write(commaSpaceBytes) + } else { + p.buf.WriteByte(' ') + } + } + if plus || goSyntax { + if f := t.Field(i); f.Name != "" { + p.buf.WriteString(f.Name) + p.buf.WriteByte(':') + } + } + p.printField(getField(v, i).Interface(), verb, plus, goSyntax, depth+1) + } + p.buf.WriteByte('}') + case *reflect.InterfaceValue: + value := f.Elem() + if value == nil { + if goSyntax { + p.buf.WriteString(reflect.Typeof(field).String()) + p.buf.Write(nilParenBytes) + } else { + p.buf.Write(nilAngleBytes) + } + } else { + return p.printField(value.Interface(), verb, plus, goSyntax, depth+1) + } + case reflect.ArrayOrSliceValue: + // Byte slices are special. + if f.Type().(reflect.ArrayOrSliceType).Elem().Kind() == reflect.Uint8 { + // We know it's a slice of bytes, but we also know it does not have static type + // []byte, or it would have been caught above. Therefore we cannot convert + // it directly in the (slightly) obvious way: f.Interface().([]byte); it doesn't have + // that type, and we can't write an expression of the right type and do a + // conversion because we don't have a static way to write the right type. + // So we build a slice by hand. This is a rare case but it would be nice + // if reflection could help a little more. + bytes := make([]byte, f.Len()) + for i := range bytes { + bytes[i] = byte(f.Elem(i).(*reflect.UintValue).Get()) + } + p.fmtBytes(bytes, verb, goSyntax, depth, field) + return verb == 's' + } + if goSyntax { + p.buf.WriteString(reflect.Typeof(field).String()) + p.buf.WriteByte('{') + } else { + p.buf.WriteByte('[') + } + for i := 0; i < f.Len(); i++ { + if i > 0 { + if goSyntax { + p.buf.Write(commaSpaceBytes) + } else { + p.buf.WriteByte(' ') + } + } + p.printField(f.Elem(i).Interface(), verb, plus, goSyntax, depth+1) + } + if goSyntax { + p.buf.WriteByte('}') + } else { + p.buf.WriteByte(']') + } + case *reflect.PtrValue: + v := f.Get() + // pointer to array or slice or struct? ok at top level + // but not embedded (avoid loops) + if v != 0 && depth == 0 { + switch a := f.Elem().(type) { + case reflect.ArrayOrSliceValue: + p.buf.WriteByte('&') + p.printField(a.Interface(), verb, plus, goSyntax, depth+1) + break BigSwitch + case *reflect.StructValue: + p.buf.WriteByte('&') + p.printField(a.Interface(), verb, plus, goSyntax, depth+1) + break BigSwitch + } + } + if goSyntax { + p.buf.WriteByte('(') + p.buf.WriteString(reflect.Typeof(field).String()) + p.buf.WriteByte(')') + p.buf.WriteByte('(') + if v == 0 { + p.buf.Write(nilBytes) + } else { + p.fmt0x64(uint64(v)) + } + p.buf.WriteByte(')') + break + } + if v == 0 { + p.buf.Write(nilAngleBytes) + break + } + p.fmt0x64(uint64(v)) + case uintptrGetter: + p.fmtPointer(field, value, verb, goSyntax) + default: + p.unknownType(f) + } + return false +} + +// intFromArg gets the fieldnumth element of a. On return, isInt reports whether the argument has type int. +func intFromArg(a []interface{}, end, i, fieldnum int) (num int, isInt bool, newi, newfieldnum int) { + newi, newfieldnum = end, fieldnum + if i < end && fieldnum < len(a) { + num, isInt = a[fieldnum].(int) + newi, newfieldnum = i+1, fieldnum+1 + } + return +} + +func (p *pp) doPrintf(format string, a []interface{}) { + end := len(format) + fieldnum := 0 // we process one field per non-trivial format + for i := 0; i < end; { + lasti := i + for i < end && format[i] != '%' { + i++ + } + if i > lasti { + p.buf.WriteString(format[lasti:i]) + } + if i >= end { + // done processing format string + break + } + + // Process one verb + i++ + // flags and widths + p.fmt.clearflags() + F: + for ; i < end; i++ { + switch format[i] { + case '#': + p.fmt.sharp = true + case '0': + p.fmt.zero = true + case '+': + p.fmt.plus = true + case '-': + p.fmt.minus = true + case ' ': + p.fmt.space = true + default: + break F + } + } + // do we have width? + if i < end && format[i] == '*' { + p.fmt.wid, p.fmt.widPresent, i, fieldnum = intFromArg(a, end, i, fieldnum) + if !p.fmt.widPresent { + p.buf.Write(widthBytes) + } + } else { + p.fmt.wid, p.fmt.widPresent, i = parsenum(format, i, end) + } + // do we have precision? + if i < end && format[i] == '.' { + if format[i+1] == '*' { + p.fmt.prec, p.fmt.precPresent, i, fieldnum = intFromArg(a, end, i+1, fieldnum) + if !p.fmt.precPresent { + p.buf.Write(precBytes) + } + } else { + p.fmt.prec, p.fmt.precPresent, i = parsenum(format, i+1, end) + } + } + if i >= end { + p.buf.Write(noVerbBytes) + continue + } + c, w := utf8.DecodeRuneInString(format[i:]) + i += w + // percent is special - absorbs no operand + if c == '%' { + p.buf.WriteByte('%') // We ignore width and prec. + continue + } + if fieldnum >= len(a) { // out of operands + p.buf.WriteByte('%') + p.add(c) + p.buf.Write(missingBytes) + continue + } + field := a[fieldnum] + fieldnum++ + + goSyntax := c == 'v' && p.fmt.sharp + plus := c == 'v' && p.fmt.plus + p.printField(field, c, plus, goSyntax, 0) + } + + if fieldnum < len(a) { + p.buf.Write(extraBytes) + for ; fieldnum < len(a); fieldnum++ { + field := a[fieldnum] + if field != nil { + p.buf.WriteString(reflect.Typeof(field).String()) + p.buf.WriteByte('=') + } + p.printField(field, 'v', false, false, 0) + if fieldnum+1 < len(a) { + p.buf.Write(commaSpaceBytes) + } + } + p.buf.WriteByte(')') + } +} + +func (p *pp) doPrint(a []interface{}, addspace, addnewline bool) { + prevString := false + for fieldnum := 0; fieldnum < len(a); fieldnum++ { + p.fmt.clearflags() + // always add spaces if we're doing println + field := a[fieldnum] + if fieldnum > 0 { + isString := field != nil && reflect.Typeof(field).Kind() == reflect.String + if addspace || !isString && !prevString { + p.buf.WriteByte(' ') + } + } + prevString = p.printField(field, 'v', false, false, 0) + } + if addnewline { + p.buf.WriteByte('\n') + } +} diff --git a/libgo/go/fmt/scan.go b/libgo/go/fmt/scan.go new file mode 100644 index 000000000..ebbb17155 --- /dev/null +++ b/libgo/go/fmt/scan.go @@ -0,0 +1,985 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fmt + +import ( + "bytes" + "io" + "os" + "reflect" + "strconv" + "strings" + "unicode" + "utf8" +) + +// readRuner is the interface to something that can read runes. If +// the object provided to Scan does not satisfy this interface, the +// object will be wrapped by a readRune object. +type readRuner interface { + ReadRune() (rune int, size int, err os.Error) +} + +// unreadRuner is the interface to something that can unread runes. +// If the object provided to Scan does not satisfy this interface, +// a local buffer will be used to back up the input, but its contents +// will be lost when Scan returns. +type unreadRuner interface { + UnreadRune() os.Error +} + +// ScanState represents the scanner state passed to custom scanners. +// Scanners may do rune-at-a-time scanning or ask the ScanState +// to discover the next space-delimited token. +type ScanState interface { + // GetRune reads the next rune (Unicode code point) from the input. + GetRune() (rune int, err os.Error) + // UngetRune causes the next call to GetRune to return the rune. + UngetRune() + // Width returns the value of the width option and whether it has been set. + // The unit is Unicode code points. + Width() (wid int, ok bool) + // Token returns the next space-delimited token from the input. If + // a width has been specified, the returned token will be no longer + // than the width. + Token() (token string, err os.Error) +} + +// Scanner is implemented by any value that has a Scan method, which scans +// the input for the representation of a value and stores the result in the +// receiver, which must be a pointer to be useful. The Scan method is called +// for any argument to Scan or Scanln that implements it. +type Scanner interface { + Scan(state ScanState, verb int) os.Error +} + +// Scan scans text read from standard input, storing successive +// space-separated values into successive arguments. Newlines count +// as space. It returns the number of items successfully scanned. +// If that is less than the number of arguments, err will report why. +func Scan(a ...interface{}) (n int, err os.Error) { + return Fscan(os.Stdin, a...) +} + +// Scanln is similar to Scan, but stops scanning at a newline and +// after the final item there must be a newline or EOF. +func Scanln(a ...interface{}) (n int, err os.Error) { + return Fscanln(os.Stdin, a...) +} + +// Scanf scans text read from standard input, storing successive +// space-separated values into successive arguments as determined by +// the format. It returns the number of items successfully scanned. +func Scanf(format string, a ...interface{}) (n int, err os.Error) { + return Fscanf(os.Stdin, format, a...) +} + +// Sscan scans the argument string, storing successive space-separated +// values into successive arguments. Newlines count as space. It +// returns the number of items successfully scanned. If that is less +// than the number of arguments, err will report why. +func Sscan(str string, a ...interface{}) (n int, err os.Error) { + return Fscan(strings.NewReader(str), a...) +} + +// Sscanln is similar to Sscan, but stops scanning at a newline and +// after the final item there must be a newline or EOF. +func Sscanln(str string, a ...interface{}) (n int, err os.Error) { + return Fscanln(strings.NewReader(str), a...) +} + +// Sscanf scans the argument string, storing successive space-separated +// values into successive arguments as determined by the format. It +// returns the number of items successfully parsed. +func Sscanf(str string, format string, a ...interface{}) (n int, err os.Error) { + return Fscanf(strings.NewReader(str), format, a...) +} + +// Fscan scans text read from r, storing successive space-separated +// values into successive arguments. Newlines count as space. It +// returns the number of items successfully scanned. If that is less +// than the number of arguments, err will report why. +func Fscan(r io.Reader, a ...interface{}) (n int, err os.Error) { + s := newScanState(r, true) + n, err = s.doScan(a) + s.free() + return +} + +// Fscanln is similar to Fscan, but stops scanning at a newline and +// after the final item there must be a newline or EOF. +func Fscanln(r io.Reader, a ...interface{}) (n int, err os.Error) { + s := newScanState(r, false) + n, err = s.doScan(a) + s.free() + return +} + +// Fscanf scans text read from r, storing successive space-separated +// values into successive arguments as determined by the format. It +// returns the number of items successfully parsed. +func Fscanf(r io.Reader, format string, a ...interface{}) (n int, err os.Error) { + s := newScanState(r, false) + n, err = s.doScanf(format, a) + s.free() + return +} + +// scanError represents an error generated by the scanning software. +// It's used as a unique signature to identify such errors when recovering. +type scanError struct { + err os.Error +} + +const EOF = -1 + +// ss is the internal implementation of ScanState. +type ss struct { + rr readRuner // where to read input + buf bytes.Buffer // token accumulator + nlIsSpace bool // whether newline counts as white space + peekRune int // one-rune lookahead + prevRune int // last rune returned by GetRune + atEOF bool // already read EOF + maxWid int // max width of field, in runes + widPresent bool // width was specified + wid int // width consumed so far; used in accept() +} + +func (s *ss) GetRune() (rune int, err os.Error) { + if s.peekRune >= 0 { + rune = s.peekRune + s.prevRune = rune + s.peekRune = -1 + return + } + rune, _, err = s.rr.ReadRune() + if err == nil { + s.prevRune = rune + } + return +} + +func (s *ss) Width() (wid int, ok bool) { + return s.maxWid, s.widPresent +} + +// The public method returns an error; this private one panics. +// If getRune reaches EOF, the return value is EOF (-1). +func (s *ss) getRune() (rune int) { + if s.atEOF { + return EOF + } + if s.peekRune >= 0 { + rune = s.peekRune + s.prevRune = rune + s.peekRune = -1 + return + } + rune, _, err := s.rr.ReadRune() + if err == nil { + s.prevRune = rune + } else if err != nil { + if err == os.EOF { + s.atEOF = true + return EOF + } + s.error(err) + } + return +} + +// mustGetRune turns os.EOF into a panic(io.ErrUnexpectedEOF). +// It is called in cases such as string scanning where an EOF is a +// syntax error. +func (s *ss) mustGetRune() (rune int) { + if s.atEOF { + s.error(io.ErrUnexpectedEOF) + } + if s.peekRune >= 0 { + rune = s.peekRune + s.peekRune = -1 + return + } + rune, _, err := s.rr.ReadRune() + if err != nil { + if err == os.EOF { + err = io.ErrUnexpectedEOF + } + s.error(err) + } + return +} + + +func (s *ss) UngetRune() { + if u, ok := s.rr.(unreadRuner); ok { + u.UnreadRune() + } else { + s.peekRune = s.prevRune + } +} + +func (s *ss) error(err os.Error) { + panic(scanError{err}) +} + +func (s *ss) errorString(err string) { + panic(scanError{os.ErrorString(err)}) +} + +func (s *ss) Token() (tok string, err os.Error) { + defer func() { + if e := recover(); e != nil { + if se, ok := e.(scanError); ok { + err = se.err + } else { + panic(e) + } + } + }() + tok = s.token() + return +} + +// readRune is a structure to enable reading UTF-8 encoded code points +// from an io.Reader. It is used if the Reader given to the scanner does +// not already implement ReadRuner. +type readRune struct { + reader io.Reader + buf [utf8.UTFMax]byte // used only inside ReadRune + pending int // number of bytes in pendBuf; only >0 for bad UTF-8 + pendBuf [utf8.UTFMax]byte // bytes left over +} + +// readByte returns the next byte from the input, which may be +// left over from a previous read if the UTF-8 was ill-formed. +func (r *readRune) readByte() (b byte, err os.Error) { + if r.pending > 0 { + b = r.pendBuf[0] + copy(r.pendBuf[0:], r.pendBuf[1:]) + r.pending-- + return + } + _, err = r.reader.Read(r.pendBuf[0:1]) + return r.pendBuf[0], err +} + +// unread saves the bytes for the next read. +func (r *readRune) unread(buf []byte) { + copy(r.pendBuf[r.pending:], buf) + r.pending += len(buf) +} + +// ReadRune returns the next UTF-8 encoded code point from the +// io.Reader inside r. +func (r *readRune) ReadRune() (rune int, size int, err os.Error) { + r.buf[0], err = r.readByte() + if err != nil { + return 0, 0, err + } + if r.buf[0] < utf8.RuneSelf { // fast check for common ASCII case + rune = int(r.buf[0]) + return + } + var n int + for n = 1; !utf8.FullRune(r.buf[0:n]); n++ { + r.buf[n], err = r.readByte() + if err != nil { + if err == os.EOF { + err = nil + break + } + return + } + } + rune, size = utf8.DecodeRune(r.buf[0:n]) + if size < n { // an error + r.unread(r.buf[size:n]) + } + return +} + + +// A leaky bucket of reusable ss structures. +var ssFree = make(chan *ss, 100) + +// Allocate a new ss struct. Probably can grab the previous one from ssFree. +func newScanState(r io.Reader, nlIsSpace bool) *ss { + s, ok := <-ssFree + if !ok { + s = new(ss) + } + if rr, ok := r.(readRuner); ok { + s.rr = rr + } else { + s.rr = &readRune{reader: r} + } + s.nlIsSpace = nlIsSpace + s.peekRune = -1 + s.atEOF = false + s.maxWid = 0 + s.widPresent = false + return s +} + +// Save used ss structs in ssFree; avoid an allocation per invocation. +func (s *ss) free() { + // Don't hold on to ss structs with large buffers. + if cap(s.buf.Bytes()) > 1024 { + return + } + s.buf.Reset() + s.rr = nil + _ = ssFree <- s +} + +// skipSpace skips spaces and maybe newlines. +func (s *ss) skipSpace(stopAtNewline bool) { + for { + rune := s.getRune() + if rune == EOF { + return + } + if rune == '\n' { + if stopAtNewline { + break + } + if s.nlIsSpace { + continue + } + s.errorString("unexpected newline") + return + } + if !unicode.IsSpace(rune) { + s.UngetRune() + break + } + } +} + +// token returns the next space-delimited string from the input. It +// skips white space. For Scanln, it stops at newlines. For Scan, +// newlines are treated as spaces. +func (s *ss) token() string { + s.skipSpace(false) + // read until white space or newline + for nrunes := 0; !s.widPresent || nrunes < s.maxWid; nrunes++ { + rune := s.getRune() + if rune == EOF { + break + } + if unicode.IsSpace(rune) { + s.UngetRune() + break + } + s.buf.WriteRune(rune) + } + return s.buf.String() +} + +// typeError indicates that the type of the operand did not match the format +func (s *ss) typeError(field interface{}, expected string) { + s.errorString("expected field of type pointer to " + expected + "; found " + reflect.Typeof(field).String()) +} + +var complexError = os.ErrorString("syntax error scanning complex number") +var boolError = os.ErrorString("syntax error scanning boolean") + +// consume reads the next rune in the input and reports whether it is in the ok string. +// If accept is true, it puts the character into the input token. +func (s *ss) consume(ok string, accept bool) bool { + if s.wid >= s.maxWid { + return false + } + rune := s.getRune() + if rune == EOF { + return false + } + for i := 0; i < len(ok); i++ { + if int(ok[i]) == rune { + if accept { + s.buf.WriteRune(rune) + s.wid++ + } + return true + } + } + if rune != EOF && accept { + s.UngetRune() + } + return false +} + +// accept checks the next rune in the input. If it's a byte (sic) in the string, it puts it in the +// buffer and returns true. Otherwise it return false. +func (s *ss) accept(ok string) bool { + return s.consume(ok, true) +} + +// okVerb verifies that the verb is present in the list, setting s.err appropriately if not. +func (s *ss) okVerb(verb int, okVerbs, typ string) bool { + for _, v := range okVerbs { + if v == verb { + return true + } + } + s.errorString("bad verb %" + string(verb) + " for " + typ) + return false +} + +// scanBool returns the value of the boolean represented by the next token. +func (s *ss) scanBool(verb int) bool { + if !s.okVerb(verb, "tv", "boolean") { + return false + } + // Syntax-checking a boolean is annoying. We're not fastidious about case. + switch s.mustGetRune() { + case '0': + return false + case '1': + return true + case 't', 'T': + if s.accept("rR") && (!s.accept("uU") || !s.accept("eE")) { + s.error(boolError) + } + return true + case 'f', 'F': + if s.accept("aL") && (!s.accept("lL") || !s.accept("sS") || !s.accept("eE")) { + s.error(boolError) + } + return false + } + return false +} + +// Numerical elements +const ( + binaryDigits = "01" + octalDigits = "01234567" + decimalDigits = "0123456789" + hexadecimalDigits = "0123456789aAbBcCdDeEfF" + sign = "+-" + period = "." + exponent = "eE" +) + +// getBase returns the numeric base represented by the verb and its digit string. +func (s *ss) getBase(verb int) (base int, digits string) { + s.okVerb(verb, "bdoUxXv", "integer") // sets s.err + base = 10 + digits = decimalDigits + switch verb { + case 'b': + base = 2 + digits = binaryDigits + case 'o': + base = 8 + digits = octalDigits + case 'x', 'X', 'U': + base = 16 + digits = hexadecimalDigits + } + return +} + +// scanNumber returns the numerical string with specified digits starting here. +func (s *ss) scanNumber(digits string) string { + if !s.accept(digits) { + s.errorString("expected integer") + } + for s.accept(digits) { + } + return s.buf.String() +} + +// scanRune returns the next rune value in the input. +func (s *ss) scanRune(bitSize int) int64 { + rune := int64(s.mustGetRune()) + n := uint(bitSize) + x := (rune << (64 - n)) >> (64 - n) + if x != rune { + s.errorString("overflow on character value " + string(rune)) + } + return rune +} + +// scanInt returns the value of the integer represented by the next +// token, checking for overflow. Any error is stored in s.err. +func (s *ss) scanInt(verb int, bitSize int) int64 { + if verb == 'c' { + return s.scanRune(bitSize) + } + base, digits := s.getBase(verb) + s.skipSpace(false) + if verb == 'U' { + if !s.consume("U", false) || !s.consume("+", false) { + s.errorString("bad unicode format ") + } + } else { + s.accept(sign) // If there's a sign, it will be left in the token buffer. + } + tok := s.scanNumber(digits) + i, err := strconv.Btoi64(tok, base) + if err != nil { + s.error(err) + } + n := uint(bitSize) + x := (i << (64 - n)) >> (64 - n) + if x != i { + s.errorString("integer overflow on token " + tok) + } + return i +} + +// scanUint returns the value of the unsigned integer represented +// by the next token, checking for overflow. Any error is stored in s.err. +func (s *ss) scanUint(verb int, bitSize int) uint64 { + if verb == 'c' { + return uint64(s.scanRune(bitSize)) + } + base, digits := s.getBase(verb) + s.skipSpace(false) + if verb == 'U' { + if !s.consume("U", false) || !s.consume("+", false) { + s.errorString("bad unicode format ") + } + } + tok := s.scanNumber(digits) + i, err := strconv.Btoui64(tok, base) + if err != nil { + s.error(err) + } + n := uint(bitSize) + x := (i << (64 - n)) >> (64 - n) + if x != i { + s.errorString("unsigned integer overflow on token " + tok) + } + return i +} + +// floatToken returns the floating-point number starting here, no longer than swid +// if the width is specified. It's not rigorous about syntax because it doesn't check that +// we have at least some digits, but Atof will do that. +func (s *ss) floatToken() string { + s.buf.Reset() + // NaN? + if s.accept("nN") && s.accept("aA") && s.accept("nN") { + return s.buf.String() + } + // leading sign? + s.accept(sign) + // Inf? + if s.accept("iI") && s.accept("nN") && s.accept("fF") { + return s.buf.String() + } + // digits? + for s.accept(decimalDigits) { + } + // decimal point? + if s.accept(period) { + // fraction? + for s.accept(decimalDigits) { + } + } + // exponent? + if s.accept(exponent) { + // leading sign? + s.accept(sign) + // digits? + for s.accept(decimalDigits) { + } + } + return s.buf.String() +} + +// complexTokens returns the real and imaginary parts of the complex number starting here. +// The number might be parenthesized and has the format (N+Ni) where N is a floating-point +// number and there are no spaces within. +func (s *ss) complexTokens() (real, imag string) { + // TODO: accept N and Ni independently? + parens := s.accept("(") + real = s.floatToken() + s.buf.Reset() + // Must now have a sign. + if !s.accept("+-") { + s.error(complexError) + } + // Sign is now in buffer + imagSign := s.buf.String() + imag = s.floatToken() + if !s.accept("i") { + s.error(complexError) + } + if parens && !s.accept(")") { + s.error(complexError) + } + return real, imagSign + imag +} + +// convertFloat converts the string to a float64value. +func (s *ss) convertFloat(str string, n int) float64 { + f, err := strconv.AtofN(str, n) + if err != nil { + s.error(err) + } + return f +} + +// convertComplex converts the next token to a complex128 value. +// The atof argument is a type-specific reader for the underlying type. +// If we're reading complex64, atof will parse float32s and convert them +// to float64's to avoid reproducing this code for each complex type. +func (s *ss) scanComplex(verb int, n int) complex128 { + if !s.okVerb(verb, floatVerbs, "complex") { + return 0 + } + s.skipSpace(false) + sreal, simag := s.complexTokens() + real := s.convertFloat(sreal, n/2) + imag := s.convertFloat(simag, n/2) + return complex(real, imag) +} + +// convertString returns the string represented by the next input characters. +// The format of the input is determined by the verb. +func (s *ss) convertString(verb int) (str string) { + if !s.okVerb(verb, "svqx", "string") { + return "" + } + s.skipSpace(false) + switch verb { + case 'q': + str = s.quotedString() + case 'x': + str = s.hexString() + default: + str = s.token() // %s and %v just return the next word + } + // Empty strings other than with %q are not OK. + if len(str) == 0 && verb != 'q' && s.maxWid > 0 { + s.errorString("Scan: no data for string") + } + return +} + +// quotedString returns the double- or back-quoted string represented by the next input characters. +func (s *ss) quotedString() string { + quote := s.mustGetRune() + switch quote { + case '`': + // Back-quoted: Anything goes until EOF or back quote. + for { + rune := s.mustGetRune() + if rune == quote { + break + } + s.buf.WriteRune(rune) + } + return s.buf.String() + case '"': + // Double-quoted: Include the quotes and let strconv.Unquote do the backslash escapes. + s.buf.WriteRune(quote) + for { + rune := s.mustGetRune() + s.buf.WriteRune(rune) + if rune == '\\' { + // In a legal backslash escape, no matter how long, only the character + // immediately after the escape can itself be a backslash or quote. + // Thus we only need to protect the first character after the backslash. + rune := s.mustGetRune() + s.buf.WriteRune(rune) + } else if rune == '"' { + break + } + } + result, err := strconv.Unquote(s.buf.String()) + if err != nil { + s.error(err) + } + return result + default: + s.errorString("expected quoted string") + } + return "" +} + +// hexDigit returns the value of the hexadecimal digit +func (s *ss) hexDigit(digit int) int { + switch digit { + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + return digit - '0' + case 'a', 'b', 'c', 'd', 'e', 'f': + return 10 + digit - 'a' + case 'A', 'B', 'C', 'D', 'E', 'F': + return 10 + digit - 'A' + } + s.errorString("Scan: illegal hex digit") + return 0 +} + +// hexByte returns the next hex-encoded (two-character) byte from the input. +// There must be either two hexadecimal digits or a space character in the input. +func (s *ss) hexByte() (b byte, ok bool) { + rune1 := s.getRune() + if rune1 == EOF { + return + } + if unicode.IsSpace(rune1) { + s.UngetRune() + return + } + rune2 := s.mustGetRune() + return byte(s.hexDigit(rune1)<<4 | s.hexDigit(rune2)), true +} + +// hexString returns the space-delimited hexpair-encoded string. +func (s *ss) hexString() string { + for { + b, ok := s.hexByte() + if !ok { + break + } + s.buf.WriteByte(b) + } + if s.buf.Len() == 0 { + s.errorString("Scan: no hex data for %x string") + return "" + } + return s.buf.String() +} + +const floatVerbs = "eEfFgGv" + +// scanOne scans a single value, deriving the scanner from the type of the argument. +func (s *ss) scanOne(verb int, field interface{}) { + s.buf.Reset() + var err os.Error + // If the parameter has its own Scan method, use that. + if v, ok := field.(Scanner); ok { + err = v.Scan(s, verb) + if err != nil { + s.error(err) + } + return + } + if !s.widPresent { + s.maxWid = 1 << 30 // Huge + } + s.wid = 0 + switch v := field.(type) { + case *bool: + *v = s.scanBool(verb) + case *complex64: + *v = complex64(s.scanComplex(verb, 64)) + case *complex128: + *v = s.scanComplex(verb, 128) + case *int: + *v = int(s.scanInt(verb, intBits)) + case *int8: + *v = int8(s.scanInt(verb, 8)) + case *int16: + *v = int16(s.scanInt(verb, 16)) + case *int32: + *v = int32(s.scanInt(verb, 32)) + case *int64: + *v = s.scanInt(verb, 64) + case *uint: + *v = uint(s.scanUint(verb, intBits)) + case *uint8: + *v = uint8(s.scanUint(verb, 8)) + case *uint16: + *v = uint16(s.scanUint(verb, 16)) + case *uint32: + *v = uint32(s.scanUint(verb, 32)) + case *uint64: + *v = s.scanUint(verb, 64) + case *uintptr: + *v = uintptr(s.scanUint(verb, uintptrBits)) + // Floats are tricky because you want to scan in the precision of the result, not + // scan in high precision and convert, in order to preserve the correct error condition. + case *float32: + if s.okVerb(verb, floatVerbs, "float32") { + s.skipSpace(false) + *v = float32(s.convertFloat(s.floatToken(), 32)) + } + case *float64: + if s.okVerb(verb, floatVerbs, "float64") { + s.skipSpace(false) + *v = s.convertFloat(s.floatToken(), 64) + } + case *string: + *v = s.convertString(verb) + case *[]byte: + // We scan to string and convert so we get a copy of the data. + // If we scanned to bytes, the slice would point at the buffer. + *v = []byte(s.convertString(verb)) + default: + val := reflect.NewValue(v) + ptr, ok := val.(*reflect.PtrValue) + if !ok { + s.errorString("Scan: type not a pointer: " + val.Type().String()) + return + } + switch v := ptr.Elem().(type) { + case *reflect.BoolValue: + v.Set(s.scanBool(verb)) + case *reflect.IntValue: + v.Set(s.scanInt(verb, v.Type().Bits())) + case *reflect.UintValue: + v.Set(s.scanUint(verb, v.Type().Bits())) + case *reflect.StringValue: + v.Set(s.convertString(verb)) + case *reflect.SliceValue: + // For now, can only handle (renamed) []byte. + typ := v.Type().(*reflect.SliceType) + if typ.Elem().Kind() != reflect.Uint8 { + goto CantHandle + } + str := s.convertString(verb) + v.Set(reflect.MakeSlice(typ, len(str), len(str))) + for i := 0; i < len(str); i++ { + v.Elem(i).(*reflect.UintValue).Set(uint64(str[i])) + } + case *reflect.FloatValue: + s.skipSpace(false) + v.Set(s.convertFloat(s.floatToken(), v.Type().Bits())) + case *reflect.ComplexValue: + v.Set(s.scanComplex(verb, v.Type().Bits())) + default: + CantHandle: + s.errorString("Scan: can't handle type: " + val.Type().String()) + } + } +} + +// errorHandler turns local panics into error returns. EOFs are benign. +func errorHandler(errp *os.Error) { + if e := recover(); e != nil { + if se, ok := e.(scanError); ok { // catch local error + if se.err != os.EOF { + *errp = se.err + } + } else { + panic(e) + } + } +} + +// doScan does the real work for scanning without a format string. +// At the moment, it handles only pointers to basic types. +func (s *ss) doScan(a []interface{}) (numProcessed int, err os.Error) { + defer errorHandler(&err) + for _, field := range a { + s.scanOne('v', field) + numProcessed++ + } + // Check for newline if required. + if !s.nlIsSpace { + for { + rune := s.getRune() + if rune == '\n' || rune == EOF { + break + } + if !unicode.IsSpace(rune) { + s.errorString("Scan: expected newline") + break + } + } + } + return +} + +// advance determines whether the next characters in the input match +// those of the format. It returns the number of bytes (sic) consumed +// in the format. Newlines included, all runs of space characters in +// either input or format behave as a single space. This routine also +// handles the %% case. If the return value is zero, either format +// starts with a % (with no following %) or the input is empty. +// If it is negative, the input did not match the string. +func (s *ss) advance(format string) (i int) { + for i < len(format) { + fmtc, w := utf8.DecodeRuneInString(format[i:]) + if fmtc == '%' { + // %% acts like a real percent + nextc, _ := utf8.DecodeRuneInString(format[i+w:]) // will not match % if string is empty + if nextc != '%' { + return + } + i += w // skip the first % + } + sawSpace := false + for unicode.IsSpace(fmtc) && i < len(format) { + sawSpace = true + i += w + fmtc, w = utf8.DecodeRuneInString(format[i:]) + } + if sawSpace { + // There was space in the format, so there should be space (EOF) + // in the input. + inputc := s.getRune() + if inputc == EOF { + return + } + if !unicode.IsSpace(inputc) { + // Space in format but not in input: error + s.errorString("expected space in input to match format") + } + s.skipSpace(true) + continue + } + inputc := s.mustGetRune() + if fmtc != inputc { + s.UngetRune() + return -1 + } + i += w + } + return +} + +// doScanf does the real work when scanning with a format string. +// At the moment, it handles only pointers to basic types. +func (s *ss) doScanf(format string, a []interface{}) (numProcessed int, err os.Error) { + defer errorHandler(&err) + end := len(format) - 1 + // We process one item per non-trivial format + for i := 0; i <= end; { + w := s.advance(format[i:]) + if w > 0 { + i += w + continue + } + // Either we failed to advance, we have a percent character, or we ran out of input. + if format[i] != '%' { + // Can't advance format. Why not? + if w < 0 { + s.errorString("input does not match format") + } + // Otherwise at EOF; "too many operands" error handled below + break + } + i++ // % is one byte + + // do we have 20 (width)? + s.maxWid, s.widPresent, i = parsenum(format, i, end) + + c, w := utf8.DecodeRuneInString(format[i:]) + i += w + + if numProcessed >= len(a) { // out of operands + s.errorString("too few operands for format %" + format[i-w:]) + break + } + field := a[numProcessed] + + s.scanOne(c, field) + numProcessed++ + } + if numProcessed < len(a) { + s.errorString("too many operands") + } + return +} diff --git a/libgo/go/fmt/scan_test.go b/libgo/go/fmt/scan_test.go new file mode 100644 index 000000000..78b9fbb4a --- /dev/null +++ b/libgo/go/fmt/scan_test.go @@ -0,0 +1,659 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fmt_test + +import ( + "bufio" + . "fmt" + "io" + "math" + "os" + "reflect" + "regexp" + "strings" + "testing" + "utf8" +) + +type ScanTest struct { + text string + in interface{} + out interface{} +} + +type ScanfTest struct { + format string + text string + in interface{} + out interface{} +} + +type ScanfMultiTest struct { + format string + text string + in []interface{} + out []interface{} + err string +} + +var ( + boolVal bool + intVal int + int8Val int8 + int16Val int16 + int32Val int32 + int64Val int64 + uintVal uint + uint8Val uint8 + uint16Val uint16 + uint32Val uint32 + uint64Val uint64 + float32Val float32 + float64Val float64 + stringVal string + stringVal1 string + bytesVal []byte + complex64Val complex64 + complex128Val complex128 + renamedBoolVal renamedBool + renamedIntVal renamedInt + renamedInt8Val renamedInt8 + renamedInt16Val renamedInt16 + renamedInt32Val renamedInt32 + renamedInt64Val renamedInt64 + renamedUintVal renamedUint + renamedUint8Val renamedUint8 + renamedUint16Val renamedUint16 + renamedUint32Val renamedUint32 + renamedUint64Val renamedUint64 + renamedUintptrVal renamedUintptr + renamedStringVal renamedString + renamedBytesVal renamedBytes + renamedFloat32Val renamedFloat32 + renamedFloat64Val renamedFloat64 + renamedComplex64Val renamedComplex64 + renamedComplex128Val renamedComplex128 +) + +type FloatTest struct { + text string + in float64 + out float64 +} + +// Xs accepts any non-empty run of the verb character +type Xs string + +func (x *Xs) Scan(state ScanState, verb int) os.Error { + var tok string + var c int + var err os.Error + wid, present := state.Width() + if !present { + tok, err = state.Token() + } else { + for i := 0; i < wid; i++ { + c, err = state.GetRune() + if err != nil { + break + } + tok += string(c) + } + } + if err != nil { + return err + } + if !regexp.MustCompile("^" + string(verb) + "+$").MatchString(tok) { + return os.ErrorString("syntax error for xs") + } + *x = Xs(tok) + return nil +} + +var xVal Xs + +// myStringReader implements Read but not ReadRune, allowing us to test our readRune wrapper +// type that creates something that can read runes given only Read(). +type myStringReader struct { + r *strings.Reader +} + +func (s *myStringReader) Read(p []byte) (n int, err os.Error) { + return s.r.Read(p) +} + +func newReader(s string) *myStringReader { + return &myStringReader{strings.NewReader(s)} +} + +var scanTests = []ScanTest{ + // Numbers + {"T\n", &boolVal, true}, // boolean test vals toggle to be sure they are written + {"F\n", &boolVal, false}, // restored to zero value + {"21\n", &intVal, 21}, + {"22\n", &int8Val, int8(22)}, + {"23\n", &int16Val, int16(23)}, + {"24\n", &int32Val, int32(24)}, + {"25\n", &int64Val, int64(25)}, + {"127\n", &int8Val, int8(127)}, + {"-21\n", &intVal, -21}, + {"-22\n", &int8Val, int8(-22)}, + {"-23\n", &int16Val, int16(-23)}, + {"-24\n", &int32Val, int32(-24)}, + {"-25\n", &int64Val, int64(-25)}, + {"-128\n", &int8Val, int8(-128)}, + {"+21\n", &intVal, +21}, + {"+22\n", &int8Val, int8(+22)}, + {"+23\n", &int16Val, int16(+23)}, + {"+24\n", &int32Val, int32(+24)}, + {"+25\n", &int64Val, int64(+25)}, + {"+127\n", &int8Val, int8(+127)}, + {"26\n", &uintVal, uint(26)}, + {"27\n", &uint8Val, uint8(27)}, + {"28\n", &uint16Val, uint16(28)}, + {"29\n", &uint32Val, uint32(29)}, + {"30\n", &uint64Val, uint64(30)}, + {"255\n", &uint8Val, uint8(255)}, + {"32767\n", &int16Val, int16(32767)}, + {"2.3\n", &float64Val, 2.3}, + {"2.3e1\n", &float32Val, float32(2.3e1)}, + {"2.3e2\n", &float64Val, 2.3e2}, + {"2.35\n", &stringVal, "2.35"}, + {"2345678\n", &bytesVal, []byte("2345678")}, + {"(3.4e1-2i)\n", &complex128Val, 3.4e1 - 2i}, + {"-3.45e1-3i\n", &complex64Val, complex64(-3.45e1 - 3i)}, + {"-.45e1-1e2i\n", &complex128Val, complex128(-.45e1 - 100i)}, + {"hello\n", &stringVal, "hello"}, + + // Renamed types + {"true\n", &renamedBoolVal, renamedBool(true)}, + {"F\n", &renamedBoolVal, renamedBool(false)}, + {"101\n", &renamedIntVal, renamedInt(101)}, + {"102\n", &renamedIntVal, renamedInt(102)}, + {"103\n", &renamedUintVal, renamedUint(103)}, + {"104\n", &renamedUintVal, renamedUint(104)}, + {"105\n", &renamedInt8Val, renamedInt8(105)}, + {"106\n", &renamedInt16Val, renamedInt16(106)}, + {"107\n", &renamedInt32Val, renamedInt32(107)}, + {"108\n", &renamedInt64Val, renamedInt64(108)}, + {"109\n", &renamedUint8Val, renamedUint8(109)}, + {"110\n", &renamedUint16Val, renamedUint16(110)}, + {"111\n", &renamedUint32Val, renamedUint32(111)}, + {"112\n", &renamedUint64Val, renamedUint64(112)}, + {"113\n", &renamedUintptrVal, renamedUintptr(113)}, + {"114\n", &renamedStringVal, renamedString("114")}, + {"115\n", &renamedBytesVal, renamedBytes([]byte("115"))}, + + // Custom scanner. + {" vvv ", &xVal, Xs("vvv")}, + + // Fixed bugs + {"2147483648\n", &int64Val, int64(2147483648)}, // was: integer overflow +} + +var scanfTests = []ScanfTest{ + {"%v", "TRUE\n", &boolVal, true}, + {"%t", "false\n", &boolVal, false}, + {"%v", "-71\n", &intVal, -71}, + {"%d", "72\n", &intVal, 72}, + {"%c", "a\n", &intVal, 'a'}, + {"%c", "\u5072\n", &intVal, 0x5072}, + {"%c", "\u1234\n", &intVal, '\u1234'}, + {"%d", "73\n", &int8Val, int8(73)}, + {"%d", "+74\n", &int16Val, int16(74)}, + {"%d", "75\n", &int32Val, int32(75)}, + {"%d", "76\n", &int64Val, int64(76)}, + {"%b", "1001001\n", &intVal, 73}, + {"%o", "075\n", &intVal, 075}, + {"%x", "a75\n", &intVal, 0xa75}, + {"%v", "71\n", &uintVal, uint(71)}, + {"%d", "72\n", &uintVal, uint(72)}, + {"%d", "73\n", &uint8Val, uint8(73)}, + {"%d", "74\n", &uint16Val, uint16(74)}, + {"%d", "75\n", &uint32Val, uint32(75)}, + {"%d", "76\n", &uint64Val, uint64(76)}, + {"%b", "1001001\n", &uintVal, uint(73)}, + {"%o", "075\n", &uintVal, uint(075)}, + {"%x", "a75\n", &uintVal, uint(0xa75)}, + {"%x", "A75\n", &uintVal, uint(0xa75)}, + {"%U", "U+1234\n", &intVal, int(0x1234)}, + {"%U", "U+4567\n", &uintVal, uint(0x4567)}, + + // Strings + {"%s", "using-%s\n", &stringVal, "using-%s"}, + {"%x", "7573696e672d2578\n", &stringVal, "using-%x"}, + {"%q", `"quoted\twith\\do\u0075bl\x65s"` + "\n", &stringVal, "quoted\twith\\doubles"}, + {"%q", "`quoted with backs`\n", &stringVal, "quoted with backs"}, + + // Byte slices + {"%s", "bytes-%s\n", &bytesVal, []byte("bytes-%s")}, + {"%x", "62797465732d2578\n", &bytesVal, []byte("bytes-%x")}, + {"%q", `"bytes\rwith\vdo\u0075bl\x65s"` + "\n", &bytesVal, []byte("bytes\rwith\vdoubles")}, + {"%q", "`bytes with backs`\n", &bytesVal, []byte("bytes with backs")}, + + // Renamed types + {"%v\n", "true\n", &renamedBoolVal, renamedBool(true)}, + {"%t\n", "F\n", &renamedBoolVal, renamedBool(false)}, + {"%v", "101\n", &renamedIntVal, renamedInt(101)}, + {"%c", "\u0101\n", &renamedIntVal, renamedInt('\u0101')}, + {"%o", "0146\n", &renamedIntVal, renamedInt(102)}, + {"%v", "103\n", &renamedUintVal, renamedUint(103)}, + {"%d", "104\n", &renamedUintVal, renamedUint(104)}, + {"%d", "105\n", &renamedInt8Val, renamedInt8(105)}, + {"%d", "106\n", &renamedInt16Val, renamedInt16(106)}, + {"%d", "107\n", &renamedInt32Val, renamedInt32(107)}, + {"%d", "108\n", &renamedInt64Val, renamedInt64(108)}, + {"%x", "6D\n", &renamedUint8Val, renamedUint8(109)}, + {"%o", "0156\n", &renamedUint16Val, renamedUint16(110)}, + {"%d", "111\n", &renamedUint32Val, renamedUint32(111)}, + {"%d", "112\n", &renamedUint64Val, renamedUint64(112)}, + {"%d", "113\n", &renamedUintptrVal, renamedUintptr(113)}, + {"%s", "114\n", &renamedStringVal, renamedString("114")}, + {"%q", "\"1155\"\n", &renamedBytesVal, renamedBytes([]byte("1155"))}, + {"%g", "116e1\n", &renamedFloat32Val, renamedFloat32(116e1)}, + {"%g", "-11.7e+1", &renamedFloat64Val, renamedFloat64(-11.7e+1)}, + {"%g", "11+6e1i\n", &renamedComplex64Val, renamedComplex64(11 + 6e1i)}, + {"%g", "-11.+7e+1i", &renamedComplex128Val, renamedComplex128(-11. + 7e+1i)}, + + // Interesting formats + {"here is\tthe value:%d", "here is the\tvalue:118\n", &intVal, 118}, + {"%% %%:%d", "% %:119\n", &intVal, 119}, + + // Corner cases + {"%x", "FFFFFFFF\n", &uint32Val, uint32(0xFFFFFFFF)}, + + // Custom scanner. + {"%s", " sss ", &xVal, Xs("sss")}, + {"%2s", "sssss", &xVal, Xs("ss")}, + + // Fixed bugs + {"%d\n", "27\n", &intVal, 27}, // ok + {"%d\n", "28 \n", &intVal, 28}, // was: "unexpected newline" +} + +var overflowTests = []ScanTest{ + {"128", &int8Val, 0}, + {"32768", &int16Val, 0}, + {"-129", &int8Val, 0}, + {"-32769", &int16Val, 0}, + {"256", &uint8Val, 0}, + {"65536", &uint16Val, 0}, + {"1e100", &float32Val, 0}, + {"1e500", &float64Val, 0}, + {"(1e100+0i)", &complex64Val, 0}, + {"(1+1e100i)", &complex64Val, 0}, + {"(1-1e500i)", &complex128Val, 0}, +} + +var i, j, k int +var f float64 +var s, t string +var c complex128 +var x, y Xs + +var multiTests = []ScanfMultiTest{ + {"", "", nil, nil, ""}, + {"%d", "23", args(&i), args(23), ""}, + {"%2s%3s", "22333", args(&s, &t), args("22", "333"), ""}, + {"%2d%3d", "44555", args(&i, &j), args(44, 555), ""}, + {"%2d.%3d", "66.777", args(&i, &j), args(66, 777), ""}, + {"%d, %d", "23, 18", args(&i, &j), args(23, 18), ""}, + {"%3d22%3d", "33322333", args(&i, &j), args(333, 333), ""}, + {"%6vX=%3fY", "3+2iX=2.5Y", args(&c, &f), args((3 + 2i), 2.5), ""}, + {"%d%s", "123abc", args(&i, &s), args(123, "abc"), ""}, + {"%c%c%c", "2\u50c2X", args(&i, &j, &k), args('2', '\u50c2', 'X'), ""}, + + // Custom scanner. + {"%2e%f", "eefffff", args(&x, &y), args(Xs("ee"), Xs("fffff")), ""}, + + // Errors + {"%t", "23 18", args(&i), nil, "bad verb"}, + {"%d %d %d", "23 18", args(&i, &j), args(23, 18), "too few operands"}, + {"%d %d", "23 18 27", args(&i, &j, &k), args(23, 18), "too many operands"}, + {"%c", "\u0100", args(&int8Val), nil, "overflow"}, + {"X%d", "10X", args(&intVal), nil, "input does not match format"}, + + // Bad UTF-8: should see every byte. + {"%c%c%c", "\xc2X\xc2", args(&i, &j, &k), args(utf8.RuneError, 'X', utf8.RuneError), ""}, +} + +func testScan(name string, t *testing.T, scan func(r io.Reader, a ...interface{}) (int, os.Error)) { + for _, test := range scanTests { + var r io.Reader + if name == "StringReader" { + r = strings.NewReader(test.text) + } else { + r = newReader(test.text) + } + n, err := scan(r, test.in) + if err != nil { + t.Errorf("%s got error scanning %q: %s", name, test.text, err) + continue + } + if n != 1 { + t.Errorf("%s count error on entry %q: got %d", name, test.text, n) + continue + } + // The incoming value may be a pointer + v := reflect.NewValue(test.in) + if p, ok := v.(*reflect.PtrValue); ok { + v = p.Elem() + } + val := v.Interface() + if !reflect.DeepEqual(val, test.out) { + t.Errorf("%s scanning %q: expected %v got %v, type %T", name, test.text, test.out, val, val) + } + } +} + +func TestScan(t *testing.T) { + testScan("StringReader", t, Fscan) +} + +func TestMyReaderScan(t *testing.T) { + testScan("myStringReader", t, Fscan) +} + +func TestScanln(t *testing.T) { + testScan("StringReader", t, Fscanln) +} + +func TestMyReaderScanln(t *testing.T) { + testScan("myStringReader", t, Fscanln) +} + +func TestScanf(t *testing.T) { + for _, test := range scanfTests { + n, err := Sscanf(test.text, test.format, test.in) + if err != nil { + t.Errorf("got error scanning (%q, %q): %s", test.format, test.text, err) + continue + } + if n != 1 { + t.Errorf("count error on entry (%q, %q): got %d", test.format, test.text, n) + continue + } + // The incoming value may be a pointer + v := reflect.NewValue(test.in) + if p, ok := v.(*reflect.PtrValue); ok { + v = p.Elem() + } + val := v.Interface() + if !reflect.DeepEqual(val, test.out) { + t.Errorf("scanning (%q, %q): expected %v got %v, type %T", test.format, test.text, test.out, val, val) + } + } +} + +func TestScanOverflow(t *testing.T) { + // different machines and different types report errors with different strings. + re := regexp.MustCompile("overflow|too large|out of range|not representable") + for _, test := range overflowTests { + _, err := Sscan(test.text, test.in) + if err == nil { + t.Errorf("expected overflow scanning %q", test.text) + continue + } + if !re.MatchString(err.String()) { + t.Errorf("expected overflow error scanning %q: %s", test.text, err) + } + } +} + +func verifyNaN(str string, t *testing.T) { + var f float64 + var f32 float32 + var f64 float64 + text := str + " " + str + " " + str + n, err := Fscan(strings.NewReader(text), &f, &f32, &f64) + if err != nil { + t.Errorf("got error scanning %q: %s", text, err) + } + if n != 3 { + t.Errorf("count error scanning %q: got %d", text, n) + } + if !math.IsNaN(float64(f)) || !math.IsNaN(float64(f32)) || !math.IsNaN(f64) { + t.Errorf("didn't get NaNs scanning %q: got %g %g %g", text, f, f32, f64) + } +} + +func TestNaN(t *testing.T) { + for _, s := range []string{"nan", "NAN", "NaN"} { + verifyNaN(s, t) + } +} + +func verifyInf(str string, t *testing.T) { + var f float64 + var f32 float32 + var f64 float64 + text := str + " " + str + " " + str + n, err := Fscan(strings.NewReader(text), &f, &f32, &f64) + if err != nil { + t.Errorf("got error scanning %q: %s", text, err) + } + if n != 3 { + t.Errorf("count error scanning %q: got %d", text, n) + } + sign := 1 + if str[0] == '-' { + sign = -1 + } + if !math.IsInf(float64(f), sign) || !math.IsInf(float64(f32), sign) || !math.IsInf(f64, sign) { + t.Errorf("didn't get right Infs scanning %q: got %g %g %g", text, f, f32, f64) + } +} + + +func TestInf(t *testing.T) { + for _, s := range []string{"inf", "+inf", "-inf", "INF", "-INF", "+INF", "Inf", "-Inf", "+Inf"} { + verifyInf(s, t) + } +} + +// TODO: there's no conversion from []T to ...T, but we can fake it. These +// functions do the faking. We index the table by the length of the param list. +var fscanf = []func(io.Reader, string, []interface{}) (int, os.Error){ + 0: func(r io.Reader, f string, i []interface{}) (int, os.Error) { return Fscanf(r, f) }, + 1: func(r io.Reader, f string, i []interface{}) (int, os.Error) { return Fscanf(r, f, i[0]) }, + 2: func(r io.Reader, f string, i []interface{}) (int, os.Error) { return Fscanf(r, f, i[0], i[1]) }, + 3: func(r io.Reader, f string, i []interface{}) (int, os.Error) { return Fscanf(r, f, i[0], i[1], i[2]) }, +} + +func testScanfMulti(name string, t *testing.T) { + sliceType := reflect.Typeof(make([]interface{}, 1)).(*reflect.SliceType) + for _, test := range multiTests { + var r io.Reader + if name == "StringReader" { + r = strings.NewReader(test.text) + } else { + r = newReader(test.text) + } + n, err := fscanf[len(test.in)](r, test.format, test.in) + if err != nil { + if test.err == "" { + t.Errorf("got error scanning (%q, %q): %q", test.format, test.text, err) + } else if strings.Index(err.String(), test.err) < 0 { + t.Errorf("got wrong error scanning (%q, %q): %q; expected %q", test.format, test.text, err, test.err) + } + continue + } + if test.err != "" { + t.Errorf("expected error %q error scanning (%q, %q)", test.err, test.format, test.text) + } + if n != len(test.out) { + t.Errorf("count error on entry (%q, %q): expected %d got %d", test.format, test.text, len(test.out), n) + continue + } + // Convert the slice of pointers into a slice of values + resultVal := reflect.MakeSlice(sliceType, n, n) + for i := 0; i < n; i++ { + v := reflect.NewValue(test.in[i]).(*reflect.PtrValue).Elem() + resultVal.Elem(i).(*reflect.InterfaceValue).Set(v) + } + result := resultVal.Interface() + if !reflect.DeepEqual(result, test.out) { + t.Errorf("scanning (%q, %q): expected %v got %v", test.format, test.text, test.out, result) + } + } +} + +func TestScanfMulti(t *testing.T) { + testScanfMulti("StringReader", t) +} + +func TestMyReaderScanfMulti(t *testing.T) { + testScanfMulti("myStringReader", t) +} + +func TestScanMultiple(t *testing.T) { + var a int + var s string + n, err := Sscan("123abc", &a, &s) + if n != 2 { + t.Errorf("Sscan count error: expected 2: got %d", n) + } + if err != nil { + t.Errorf("Sscan expected no error; got %s", err) + } + if a != 123 || s != "abc" { + t.Errorf("Sscan wrong values: got (%d %q) expected (123 \"abc\")", a, s) + } + n, err = Sscan("asdf", &s, &a) + if n != 1 { + t.Errorf("Sscan count error: expected 1: got %d", n) + } + if err == nil { + t.Errorf("Sscan expected error; got none: %s", err) + } + if s != "asdf" { + t.Errorf("Sscan wrong values: got %q expected \"asdf\"", s) + } +} + +// Empty strings are not valid input when scanning a string. +func TestScanEmpty(t *testing.T) { + var s1, s2 string + n, err := Sscan("abc", &s1, &s2) + if n != 1 { + t.Errorf("Sscan count error: expected 1: got %d", n) + } + if err == nil { + t.Error("Sscan expected error; got none") + } + if s1 != "abc" { + t.Errorf("Sscan wrong values: got %q expected \"abc\"", s1) + } + n, err = Sscan("", &s1, &s2) + if n != 0 { + t.Errorf("Sscan count error: expected 0: got %d", n) + } + if err == nil { + t.Error("Sscan expected error; got none") + } + // Quoted empty string is OK. + n, err = Sscanf(`""`, "%q", &s1) + if n != 1 { + t.Errorf("Sscanf count error: expected 1: got %d", n) + } + if err != nil { + t.Errorf("Sscanf expected no error with quoted string; got %s", err) + } +} + +func TestScanNotPointer(t *testing.T) { + r := strings.NewReader("1") + var a int + _, err := Fscan(r, a) + if err == nil { + t.Error("expected error scanning non-pointer") + } else if strings.Index(err.String(), "pointer") < 0 { + t.Errorf("expected pointer error scanning non-pointer, got: %s", err) + } +} + +func TestScanlnNoNewline(t *testing.T) { + var a int + _, err := Sscanln("1 x\n", &a) + if err == nil { + t.Error("expected error scanning string missing newline") + } else if strings.Index(err.String(), "newline") < 0 { + t.Errorf("expected newline error scanning string missing newline, got: %s", err) + } +} + +func TestScanlnWithMiddleNewline(t *testing.T) { + r := strings.NewReader("123\n456\n") + var a, b int + _, err := Fscanln(r, &a, &b) + if err == nil { + t.Error("expected error scanning string with extra newline") + } else if strings.Index(err.String(), "newline") < 0 { + t.Errorf("expected newline error scanning string with extra newline, got: %s", err) + } +} + +// Special Reader that counts reads at end of file. +type eofCounter struct { + reader *strings.Reader + eofCount int +} + +func (ec *eofCounter) Read(b []byte) (n int, err os.Error) { + n, err = ec.reader.Read(b) + if n == 0 { + ec.eofCount++ + } + return +} + +// Verify that when we scan, we see at most EOF once per call to a Scan function, +// and then only when it's really an EOF +func TestEOF(t *testing.T) { + ec := &eofCounter{strings.NewReader("123\n"), 0} + var a int + n, err := Fscanln(ec, &a) + if err != nil { + t.Error("unexpected error", err) + } + if n != 1 { + t.Error("expected to scan one item, got", n) + } + if ec.eofCount != 0 { + t.Error("expected zero EOFs", ec.eofCount) + ec.eofCount = 0 // reset for next test + } + n, err = Fscanln(ec, &a) + if err == nil { + t.Error("expected error scanning empty string") + } + if n != 0 { + t.Error("expected to scan zero items, got", n) + } + if ec.eofCount != 1 { + t.Error("expected one EOF, got", ec.eofCount) + } +} + +// Verify that, at least when using bufio, successive calls to Fscan do not lose runes. +func TestUnreadRuneWithBufio(t *testing.T) { + r := bufio.NewReader(strings.NewReader("123αb")) + var i int + var a string + n, err := Fscanf(r, "%d", &i) + if n != 1 || err != nil { + t.Errorf("reading int expected one item, no errors; got %d %q", n, err) + } + if i != 123 { + t.Errorf("expected 123; got %d", i) + } + n, err = Fscanf(r, "%s", &a) + if n != 1 || err != nil { + t.Errorf("reading string expected one item, no errors; got %d %q", n, err) + } + if a != "αb" { + t.Errorf("expected αb; got %q", a) + } +} diff --git a/libgo/go/fmt/stringer_test.go b/libgo/go/fmt/stringer_test.go new file mode 100644 index 000000000..0ca3f522d --- /dev/null +++ b/libgo/go/fmt/stringer_test.go @@ -0,0 +1,61 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fmt_test + +import ( + . "fmt" + "testing" +) + +type TI int +type TI8 int8 +type TI16 int16 +type TI32 int32 +type TI64 int64 +type TU uint +type TU8 uint8 +type TU16 uint16 +type TU32 uint32 +type TU64 uint64 +type TUI uintptr +type TF float64 +type TF32 float32 +type TF64 float64 +type TB bool +type TS string + +func (v TI) String() string { return Sprintf("I: %d", int(v)) } +func (v TI8) String() string { return Sprintf("I8: %d", int8(v)) } +func (v TI16) String() string { return Sprintf("I16: %d", int16(v)) } +func (v TI32) String() string { return Sprintf("I32: %d", int32(v)) } +func (v TI64) String() string { return Sprintf("I64: %d", int64(v)) } +func (v TU) String() string { return Sprintf("U: %d", uint(v)) } +func (v TU8) String() string { return Sprintf("U8: %d", uint8(v)) } +func (v TU16) String() string { return Sprintf("U16: %d", uint16(v)) } +func (v TU32) String() string { return Sprintf("U32: %d", uint32(v)) } +func (v TU64) String() string { return Sprintf("U64: %d", uint64(v)) } +func (v TUI) String() string { return Sprintf("UI: %d", uintptr(v)) } +func (v TF) String() string { return Sprintf("F: %f", float64(v)) } +func (v TF32) String() string { return Sprintf("F32: %f", float32(v)) } +func (v TF64) String() string { return Sprintf("F64: %f", float64(v)) } +func (v TB) String() string { return Sprintf("B: %t", bool(v)) } +func (v TS) String() string { return Sprintf("S: %q", string(v)) } + +func check(t *testing.T, got, want string) { + if got != want { + t.Error(got, "!=", want) + } +} + +func TestStringer(t *testing.T) { + s := Sprintf("%v %v %v %v %v", TI(0), TI8(1), TI16(2), TI32(3), TI64(4)) + check(t, s, "I: 0 I8: 1 I16: 2 I32: 3 I64: 4") + s = Sprintf("%v %v %v %v %v %v", TU(5), TU8(6), TU16(7), TU32(8), TU64(9), TUI(10)) + check(t, s, "U: 5 U8: 6 U16: 7 U32: 8 U64: 9 UI: 10") + s = Sprintf("%v %v %v", TF(1.0), TF32(2.0), TF64(3.0)) + check(t, s, "F: 1.000000 F32: 2.000000 F64: 3.000000") + s = Sprintf("%v %v", TB(true), TS("x")) + check(t, s, "B: true S: \"x\"") +} diff --git a/libgo/go/go/ast/ast.go b/libgo/go/go/ast/ast.go new file mode 100644 index 000000000..cf2ce36df --- /dev/null +++ b/libgo/go/go/ast/ast.go @@ -0,0 +1,959 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The AST package declares the types used to represent +// syntax trees for Go packages. +// +package ast + +import ( + "go/token" + "unicode" + "utf8" +) + + +// ---------------------------------------------------------------------------- +// Interfaces +// +// There are 3 main classes of nodes: Expressions and type nodes, +// statement nodes, and declaration nodes. The node names usually +// match the corresponding Go spec production names to which they +// correspond. The node fields correspond to the individual parts +// of the respective productions. +// +// All nodes contain position information marking the beginning of +// the corresponding source text segment; it is accessible via the +// Pos accessor method. Nodes may contain additional position info +// for language constructs where comments may be found between parts +// of the construct (typically any larger, parenthesized subpart). +// That position information is needed to properly position comments +// when printing the construct. + + +// All node types implement the Node interface. +type Node interface { + Pos() token.Pos // position of first character belonging to the node + End() token.Pos // position of first character immediately after the node +} + + +// All expression nodes implement the Expr interface. +type Expr interface { + Node + exprNode() +} + + +// All statement nodes implement the Stmt interface. +type Stmt interface { + Node + stmtNode() +} + + +// All declaration nodes implement the Decl interface. +type Decl interface { + Node + declNode() +} + + +// ---------------------------------------------------------------------------- +// Comments + +// A Comment node represents a single //-style or /*-style comment. +type Comment struct { + Slash token.Pos // position of "/" starting the comment + Text []byte // comment text (excluding '\n' for //-style comments) +} + + +func (c *Comment) Pos() token.Pos { return c.Slash } +func (c *Comment) End() token.Pos { return token.Pos(int(c.Slash) + len(c.Text)) } + + +// A CommentGroup represents a sequence of comments +// with no other tokens and no empty lines between. +// +type CommentGroup struct { + List []*Comment // len(List) > 0 +} + + +func (g *CommentGroup) Pos() token.Pos { return g.List[0].Pos() } +func (g *CommentGroup) End() token.Pos { return g.List[len(g.List)-1].End() } + + +// ---------------------------------------------------------------------------- +// Expressions and types + +// A Field represents a Field declaration list in a struct type, +// a method list in an interface type, or a parameter/result declaration +// in a signature. +// +type Field struct { + Doc *CommentGroup // associated documentation; or nil + Names []*Ident // field/method/parameter names; or nil if anonymous field + Type Expr // field/method/parameter type + Tag *BasicLit // field tag; or nil + Comment *CommentGroup // line comments; or nil +} + + +func (f *Field) Pos() token.Pos { + if len(f.Names) > 0 { + return f.Names[0].Pos() + } + return f.Type.Pos() +} + + +func (f *Field) End() token.Pos { + if f.Tag != nil { + return f.Tag.End() + } + return f.Type.End() +} + + +// A FieldList represents a list of Fields, enclosed by parentheses or braces. +type FieldList struct { + Opening token.Pos // position of opening parenthesis/brace, if any + List []*Field // field list + Closing token.Pos // position of closing parenthesis/brace, if any +} + + +func (f *FieldList) Pos() token.Pos { + if f.Opening.IsValid() { + return f.Opening + } + // the list should not be empty in this case; + // be conservative and guard against bad ASTs + if len(f.List) > 0 { + return f.List[0].Pos() + } + return token.NoPos +} + + +func (f *FieldList) End() token.Pos { + if f.Closing.IsValid() { + return f.Closing + 1 + } + // the list should not be empty in this case; + // be conservative and guard against bad ASTs + if n := len(f.List); n > 0 { + return f.List[n-1].End() + } + return token.NoPos +} + + +// NumFields returns the number of (named and anonymous fields) in a FieldList. +func (f *FieldList) NumFields() int { + n := 0 + if f != nil { + for _, g := range f.List { + m := len(g.Names) + if m == 0 { + m = 1 // anonymous field + } + n += m + } + } + return n +} + + +// An expression is represented by a tree consisting of one +// or more of the following concrete expression nodes. +// +type ( + // A BadExpr node is a placeholder for expressions containing + // syntax errors for which no correct expression nodes can be + // created. + // + BadExpr struct { + From, To token.Pos // position range of bad expression + } + + // An Ident node represents an identifier. + Ident struct { + NamePos token.Pos // identifier position + Name string // identifier name + Obj *Object // denoted object; or nil + } + + // An Ellipsis node stands for the "..." type in a + // parameter list or the "..." length in an array type. + // + Ellipsis struct { + Ellipsis token.Pos // position of "..." + Elt Expr // ellipsis element type (parameter lists only); or nil + } + + // A BasicLit node represents a literal of basic type. + BasicLit struct { + ValuePos token.Pos // literal position + Kind token.Token // token.INT, token.FLOAT, token.IMAG, token.CHAR, or token.STRING + Value []byte // literal string; e.g. 42, 0x7f, 3.14, 1e-9, 2.4i, 'a', '\x7f', "foo" or `\m\n\o` + } + + // A FuncLit node represents a function literal. + FuncLit struct { + Type *FuncType // function type + Body *BlockStmt // function body + } + + // A CompositeLit node represents a composite literal. + CompositeLit struct { + Type Expr // literal type; or nil + Lbrace token.Pos // position of "{" + Elts []Expr // list of composite elements; or nil + Rbrace token.Pos // position of "}" + } + + // A ParenExpr node represents a parenthesized expression. + ParenExpr struct { + Lparen token.Pos // position of "(" + X Expr // parenthesized expression + Rparen token.Pos // position of ")" + } + + // A SelectorExpr node represents an expression followed by a selector. + SelectorExpr struct { + X Expr // expression + Sel *Ident // field selector + } + + // An IndexExpr node represents an expression followed by an index. + IndexExpr struct { + X Expr // expression + Lbrack token.Pos // position of "[" + Index Expr // index expression + Rbrack token.Pos // position of "]" + } + + // An SliceExpr node represents an expression followed by slice indices. + SliceExpr struct { + X Expr // expression + Lbrack token.Pos // position of "[" + Low Expr // begin of slice range; or nil + High Expr // end of slice range; or nil + Rbrack token.Pos // position of "]" + } + + // A TypeAssertExpr node represents an expression followed by a + // type assertion. + // + TypeAssertExpr struct { + X Expr // expression + Type Expr // asserted type; nil means type switch X.(type) + } + + // A CallExpr node represents an expression followed by an argument list. + CallExpr struct { + Fun Expr // function expression + Lparen token.Pos // position of "(" + Args []Expr // function arguments; or nil + Ellipsis token.Pos // position of "...", if any + Rparen token.Pos // position of ")" + } + + // A StarExpr node represents an expression of the form "*" Expression. + // Semantically it could be a unary "*" expression, or a pointer type. + // + StarExpr struct { + Star token.Pos // position of "*" + X Expr // operand + } + + // A UnaryExpr node represents a unary expression. + // Unary "*" expressions are represented via StarExpr nodes. + // + UnaryExpr struct { + OpPos token.Pos // position of Op + Op token.Token // operator + X Expr // operand + } + + // A BinaryExpr node represents a binary expression. + BinaryExpr struct { + X Expr // left operand + OpPos token.Pos // position of Op + Op token.Token // operator + Y Expr // right operand + } + + // A KeyValueExpr node represents (key : value) pairs + // in composite literals. + // + KeyValueExpr struct { + Key Expr + Colon token.Pos // position of ":" + Value Expr + } +) + + +// The direction of a channel type is indicated by one +// of the following constants. +// +type ChanDir int + +const ( + SEND ChanDir = 1 << iota + RECV +) + + +// A type is represented by a tree consisting of one +// or more of the following type-specific expression +// nodes. +// +type ( + // An ArrayType node represents an array or slice type. + ArrayType struct { + Lbrack token.Pos // position of "[" + Len Expr // Ellipsis node for [...]T array types, nil for slice types + Elt Expr // element type + } + + // A StructType node represents a struct type. + StructType struct { + Struct token.Pos // position of "struct" keyword + Fields *FieldList // list of field declarations + Incomplete bool // true if (source) fields are missing in the Fields list + } + + // Pointer types are represented via StarExpr nodes. + + // A FuncType node represents a function type. + FuncType struct { + Func token.Pos // position of "func" keyword + Params *FieldList // (incoming) parameters + Results *FieldList // (outgoing) results; or nil + } + + // An InterfaceType node represents an interface type. + InterfaceType struct { + Interface token.Pos // position of "interface" keyword + Methods *FieldList // list of methods + Incomplete bool // true if (source) methods are missing in the Methods list + } + + // A MapType node represents a map type. + MapType struct { + Map token.Pos // position of "map" keyword + Key Expr + Value Expr + } + + // A ChanType node represents a channel type. + ChanType struct { + Begin token.Pos // position of "chan" keyword or "<-" (whichever comes first) + Dir ChanDir // channel direction + Value Expr // value type + } +) + + +// Pos and End implementations for expression/type nodes. +// +func (x *BadExpr) Pos() token.Pos { return x.From } +func (x *Ident) Pos() token.Pos { return x.NamePos } +func (x *Ellipsis) Pos() token.Pos { return x.Ellipsis } +func (x *BasicLit) Pos() token.Pos { return x.ValuePos } +func (x *FuncLit) Pos() token.Pos { return x.Type.Pos() } +func (x *CompositeLit) Pos() token.Pos { + if x.Type != nil { + return x.Type.Pos() + } + return x.Lbrace +} +func (x *ParenExpr) Pos() token.Pos { return x.Lparen } +func (x *SelectorExpr) Pos() token.Pos { return x.X.Pos() } +func (x *IndexExpr) Pos() token.Pos { return x.X.Pos() } +func (x *SliceExpr) Pos() token.Pos { return x.X.Pos() } +func (x *TypeAssertExpr) Pos() token.Pos { return x.X.Pos() } +func (x *CallExpr) Pos() token.Pos { return x.Fun.Pos() } +func (x *StarExpr) Pos() token.Pos { return x.Star } +func (x *UnaryExpr) Pos() token.Pos { return x.OpPos } +func (x *BinaryExpr) Pos() token.Pos { return x.X.Pos() } +func (x *KeyValueExpr) Pos() token.Pos { return x.Key.Pos() } +func (x *ArrayType) Pos() token.Pos { return x.Lbrack } +func (x *StructType) Pos() token.Pos { return x.Struct } +func (x *FuncType) Pos() token.Pos { return x.Func } +func (x *InterfaceType) Pos() token.Pos { return x.Interface } +func (x *MapType) Pos() token.Pos { return x.Map } +func (x *ChanType) Pos() token.Pos { return x.Begin } + + +func (x *BadExpr) End() token.Pos { return x.To } +func (x *Ident) End() token.Pos { return token.Pos(int(x.NamePos) + len(x.Name)) } +func (x *Ellipsis) End() token.Pos { + if x.Elt != nil { + return x.Elt.End() + } + return x.Ellipsis + 3 // len("...") +} +func (x *BasicLit) End() token.Pos { return token.Pos(int(x.ValuePos) + len(x.Value)) } +func (x *FuncLit) End() token.Pos { return x.Body.End() } +func (x *CompositeLit) End() token.Pos { return x.Rbrace + 1 } +func (x *ParenExpr) End() token.Pos { return x.Rparen + 1 } +func (x *SelectorExpr) End() token.Pos { return x.Sel.End() } +func (x *IndexExpr) End() token.Pos { return x.Rbrack + 1 } +func (x *SliceExpr) End() token.Pos { return x.Rbrack + 1 } +func (x *TypeAssertExpr) End() token.Pos { + if x.Type != nil { + return x.Type.End() + } + return x.X.End() +} +func (x *CallExpr) End() token.Pos { return x.Rparen + 1 } +func (x *StarExpr) End() token.Pos { return x.X.End() } +func (x *UnaryExpr) End() token.Pos { return x.X.End() } +func (x *BinaryExpr) End() token.Pos { return x.Y.End() } +func (x *KeyValueExpr) End() token.Pos { return x.Value.End() } +func (x *ArrayType) End() token.Pos { return x.Elt.End() } +func (x *StructType) End() token.Pos { return x.Fields.End() } +func (x *FuncType) End() token.Pos { + if x.Results != nil { + return x.Results.End() + } + return x.Params.End() +} +func (x *InterfaceType) End() token.Pos { return x.Methods.End() } +func (x *MapType) End() token.Pos { return x.Value.End() } +func (x *ChanType) End() token.Pos { return x.Value.End() } + + +// exprNode() ensures that only expression/type nodes can be +// assigned to an ExprNode. +// +func (x *BadExpr) exprNode() {} +func (x *Ident) exprNode() {} +func (x *Ellipsis) exprNode() {} +func (x *BasicLit) exprNode() {} +func (x *FuncLit) exprNode() {} +func (x *CompositeLit) exprNode() {} +func (x *ParenExpr) exprNode() {} +func (x *SelectorExpr) exprNode() {} +func (x *IndexExpr) exprNode() {} +func (x *SliceExpr) exprNode() {} +func (x *TypeAssertExpr) exprNode() {} +func (x *CallExpr) exprNode() {} +func (x *StarExpr) exprNode() {} +func (x *UnaryExpr) exprNode() {} +func (x *BinaryExpr) exprNode() {} +func (x *KeyValueExpr) exprNode() {} + +func (x *ArrayType) exprNode() {} +func (x *StructType) exprNode() {} +func (x *FuncType) exprNode() {} +func (x *InterfaceType) exprNode() {} +func (x *MapType) exprNode() {} +func (x *ChanType) exprNode() {} + + +// ---------------------------------------------------------------------------- +// Convenience functions for Idents + +var noPos token.Pos + +// NewIdent creates a new Ident without position. +// Useful for ASTs generated by code other than the Go parser. +// +func NewIdent(name string) *Ident { return &Ident{noPos, name, nil} } + + +// IsExported returns whether name is an exported Go symbol +// (i.e., whether it begins with an uppercase letter). +// +func IsExported(name string) bool { + ch, _ := utf8.DecodeRuneInString(name) + return unicode.IsUpper(ch) +} + + +// IsExported returns whether id is an exported Go symbol +// (i.e., whether it begins with an uppercase letter). +// +func (id *Ident) IsExported() bool { return IsExported(id.Name) } + + +func (id *Ident) String() string { + if id != nil { + return id.Name + } + return "" +} + + +// ---------------------------------------------------------------------------- +// Statements + +// A statement is represented by a tree consisting of one +// or more of the following concrete statement nodes. +// +type ( + // A BadStmt node is a placeholder for statements containing + // syntax errors for which no correct statement nodes can be + // created. + // + BadStmt struct { + From, To token.Pos // position range of bad statement + } + + // A DeclStmt node represents a declaration in a statement list. + DeclStmt struct { + Decl Decl + } + + // An EmptyStmt node represents an empty statement. + // The "position" of the empty statement is the position + // of the immediately preceeding semicolon. + // + EmptyStmt struct { + Semicolon token.Pos // position of preceeding ";" + } + + // A LabeledStmt node represents a labeled statement. + LabeledStmt struct { + Label *Ident + Colon token.Pos // position of ":" + Stmt Stmt + } + + // An ExprStmt node represents a (stand-alone) expression + // in a statement list. + // + ExprStmt struct { + X Expr // expression + } + + // An IncDecStmt node represents an increment or decrement statement. + IncDecStmt struct { + X Expr + TokPos token.Pos // position of Tok + Tok token.Token // INC or DEC + } + + // An AssignStmt node represents an assignment or + // a short variable declaration. + // + AssignStmt struct { + Lhs []Expr + TokPos token.Pos // position of Tok + Tok token.Token // assignment token, DEFINE + Rhs []Expr + } + + // A GoStmt node represents a go statement. + GoStmt struct { + Go token.Pos // position of "go" keyword + Call *CallExpr + } + + // A DeferStmt node represents a defer statement. + DeferStmt struct { + Defer token.Pos // position of "defer" keyword + Call *CallExpr + } + + // A ReturnStmt node represents a return statement. + ReturnStmt struct { + Return token.Pos // position of "return" keyword + Results []Expr // result expressions; or nil + } + + // A BranchStmt node represents a break, continue, goto, + // or fallthrough statement. + // + BranchStmt struct { + TokPos token.Pos // position of Tok + Tok token.Token // keyword token (BREAK, CONTINUE, GOTO, FALLTHROUGH) + Label *Ident // label name; or nil + } + + // A BlockStmt node represents a braced statement list. + BlockStmt struct { + Lbrace token.Pos // position of "{" + List []Stmt + Rbrace token.Pos // position of "}" + } + + // An IfStmt node represents an if statement. + IfStmt struct { + If token.Pos // position of "if" keyword + Init Stmt // initalization statement; or nil + Cond Expr // condition; or nil + Body *BlockStmt + Else Stmt // else branch; or nil + } + + // A CaseClause represents a case of an expression switch statement. + CaseClause struct { + Case token.Pos // position of "case" or "default" keyword + Values []Expr // nil means default case + Colon token.Pos // position of ":" + Body []Stmt // statement list; or nil + } + + // A SwitchStmt node represents an expression switch statement. + SwitchStmt struct { + Switch token.Pos // position of "switch" keyword + Init Stmt // initalization statement; or nil + Tag Expr // tag expression; or nil + Body *BlockStmt // CaseClauses only + } + + // A TypeCaseClause represents a case of a type switch statement. + TypeCaseClause struct { + Case token.Pos // position of "case" or "default" keyword + Types []Expr // nil means default case + Colon token.Pos // position of ":" + Body []Stmt // statement list; or nil + } + + // An TypeSwitchStmt node represents a type switch statement. + TypeSwitchStmt struct { + Switch token.Pos // position of "switch" keyword + Init Stmt // initalization statement; or nil + Assign Stmt // x := y.(type) + Body *BlockStmt // TypeCaseClauses only + } + + // A CommClause node represents a case of a select statement. + CommClause struct { + Case token.Pos // position of "case" or "default" keyword + Tok token.Token // ASSIGN or DEFINE (valid only if Lhs != nil) + Lhs, Rhs Expr // Rhs == nil means default case + Colon token.Pos // position of ":" + Body []Stmt // statement list; or nil + } + + // An SelectStmt node represents a select statement. + SelectStmt struct { + Select token.Pos // position of "select" keyword + Body *BlockStmt // CommClauses only + } + + // A ForStmt represents a for statement. + ForStmt struct { + For token.Pos // position of "for" keyword + Init Stmt // initalization statement; or nil + Cond Expr // condition; or nil + Post Stmt // post iteration statement; or nil + Body *BlockStmt + } + + // A RangeStmt represents a for statement with a range clause. + RangeStmt struct { + For token.Pos // position of "for" keyword + Key, Value Expr // Value may be nil + TokPos token.Pos // position of Tok + Tok token.Token // ASSIGN, DEFINE + X Expr // value to range over + Body *BlockStmt + } +) + + +// Pos and End implementations for statement nodes. +// +func (s *BadStmt) Pos() token.Pos { return s.From } +func (s *DeclStmt) Pos() token.Pos { return s.Decl.Pos() } +func (s *EmptyStmt) Pos() token.Pos { return s.Semicolon } +func (s *LabeledStmt) Pos() token.Pos { return s.Label.Pos() } +func (s *ExprStmt) Pos() token.Pos { return s.X.Pos() } +func (s *IncDecStmt) Pos() token.Pos { return s.X.Pos() } +func (s *AssignStmt) Pos() token.Pos { return s.Lhs[0].Pos() } +func (s *GoStmt) Pos() token.Pos { return s.Go } +func (s *DeferStmt) Pos() token.Pos { return s.Defer } +func (s *ReturnStmt) Pos() token.Pos { return s.Return } +func (s *BranchStmt) Pos() token.Pos { return s.TokPos } +func (s *BlockStmt) Pos() token.Pos { return s.Lbrace } +func (s *IfStmt) Pos() token.Pos { return s.If } +func (s *CaseClause) Pos() token.Pos { return s.Case } +func (s *SwitchStmt) Pos() token.Pos { return s.Switch } +func (s *TypeCaseClause) Pos() token.Pos { return s.Case } +func (s *TypeSwitchStmt) Pos() token.Pos { return s.Switch } +func (s *CommClause) Pos() token.Pos { return s.Case } +func (s *SelectStmt) Pos() token.Pos { return s.Select } +func (s *ForStmt) Pos() token.Pos { return s.For } +func (s *RangeStmt) Pos() token.Pos { return s.For } + + +func (s *BadStmt) End() token.Pos { return s.To } +func (s *DeclStmt) End() token.Pos { return s.Decl.End() } +func (s *EmptyStmt) End() token.Pos { + return s.Semicolon + 1 /* len(";") */ +} +func (s *LabeledStmt) End() token.Pos { return s.Stmt.End() } +func (s *ExprStmt) End() token.Pos { return s.X.End() } +func (s *IncDecStmt) End() token.Pos { + return s.TokPos + 2 /* len("++") */ +} +func (s *AssignStmt) End() token.Pos { return s.Rhs[len(s.Rhs)-1].End() } +func (s *GoStmt) End() token.Pos { return s.Call.End() } +func (s *DeferStmt) End() token.Pos { return s.Call.End() } +func (s *ReturnStmt) End() token.Pos { + if n := len(s.Results); n > 0 { + return s.Results[n-1].End() + } + return s.Return + 6 // len("return") +} +func (s *BranchStmt) End() token.Pos { + if s.Label != nil { + return s.Label.End() + } + return token.Pos(int(s.TokPos) + len(s.Tok.String())) +} +func (s *BlockStmt) End() token.Pos { return s.Rbrace + 1 } +func (s *IfStmt) End() token.Pos { + if s.Else != nil { + return s.Else.End() + } + return s.Body.End() +} +func (s *CaseClause) End() token.Pos { + if n := len(s.Body); n > 0 { + return s.Body[n-1].End() + } + return s.Colon + 1 +} +func (s *SwitchStmt) End() token.Pos { return s.Body.End() } +func (s *TypeCaseClause) End() token.Pos { + if n := len(s.Body); n > 0 { + return s.Body[n-1].End() + } + return s.Colon + 1 +} +func (s *TypeSwitchStmt) End() token.Pos { return s.Body.End() } +func (s *CommClause) End() token.Pos { + if n := len(s.Body); n > 0 { + return s.Body[n-1].End() + } + return s.Colon + 1 +} +func (s *SelectStmt) End() token.Pos { return s.Body.End() } +func (s *ForStmt) End() token.Pos { return s.Body.End() } +func (s *RangeStmt) End() token.Pos { return s.Body.End() } + + +// stmtNode() ensures that only statement nodes can be +// assigned to a StmtNode. +// +func (s *BadStmt) stmtNode() {} +func (s *DeclStmt) stmtNode() {} +func (s *EmptyStmt) stmtNode() {} +func (s *LabeledStmt) stmtNode() {} +func (s *ExprStmt) stmtNode() {} +func (s *IncDecStmt) stmtNode() {} +func (s *AssignStmt) stmtNode() {} +func (s *GoStmt) stmtNode() {} +func (s *DeferStmt) stmtNode() {} +func (s *ReturnStmt) stmtNode() {} +func (s *BranchStmt) stmtNode() {} +func (s *BlockStmt) stmtNode() {} +func (s *IfStmt) stmtNode() {} +func (s *CaseClause) stmtNode() {} +func (s *SwitchStmt) stmtNode() {} +func (s *TypeCaseClause) stmtNode() {} +func (s *TypeSwitchStmt) stmtNode() {} +func (s *CommClause) stmtNode() {} +func (s *SelectStmt) stmtNode() {} +func (s *ForStmt) stmtNode() {} +func (s *RangeStmt) stmtNode() {} + + +// ---------------------------------------------------------------------------- +// Declarations + +// A Spec node represents a single (non-parenthesized) import, +// constant, type, or variable declaration. +// +type ( + // The Spec type stands for any of *ImportSpec, *ValueSpec, and *TypeSpec. + Spec interface { + Node + specNode() + } + + // An ImportSpec node represents a single package import. + ImportSpec struct { + Doc *CommentGroup // associated documentation; or nil + Name *Ident // local package name (including "."); or nil + Path *BasicLit // package path + Comment *CommentGroup // line comments; or nil + } + + // A ValueSpec node represents a constant or variable declaration + // (ConstSpec or VarSpec production). + // + ValueSpec struct { + Doc *CommentGroup // associated documentation; or nil + Names []*Ident // value names (len(Names) > 0) + Type Expr // value type; or nil + Values []Expr // initial values; or nil + Comment *CommentGroup // line comments; or nil + } + + // A TypeSpec node represents a type declaration (TypeSpec production). + TypeSpec struct { + Doc *CommentGroup // associated documentation; or nil + Name *Ident // type name + Type Expr // *Ident, *ParenExpr, *SelectorExpr, *StarExpr, or any of the *XxxTypes + Comment *CommentGroup // line comments; or nil + } +) + + +// Pos and End implementations for spec nodes. +// +func (s *ImportSpec) Pos() token.Pos { + if s.Name != nil { + return s.Name.Pos() + } + return s.Path.Pos() +} +func (s *ValueSpec) Pos() token.Pos { return s.Names[0].Pos() } +func (s *TypeSpec) Pos() token.Pos { return s.Name.Pos() } + + +func (s *ImportSpec) End() token.Pos { return s.Path.End() } +func (s *ValueSpec) End() token.Pos { + if n := len(s.Values); n > 0 { + return s.Values[n-1].End() + } + if s.Type != nil { + return s.Type.End() + } + return s.Names[len(s.Names)-1].End() +} +func (s *TypeSpec) End() token.Pos { return s.Type.End() } + + +// specNode() ensures that only spec nodes can be +// assigned to a Spec. +// +func (s *ImportSpec) specNode() {} +func (s *ValueSpec) specNode() {} +func (s *TypeSpec) specNode() {} + + +// A declaration is represented by one of the following declaration nodes. +// +type ( + // A BadDecl node is a placeholder for declarations containing + // syntax errors for which no correct declaration nodes can be + // created. + // + BadDecl struct { + From, To token.Pos // position range of bad declaration + } + + // A GenDecl node (generic declaration node) represents an import, + // constant, type or variable declaration. A valid Lparen position + // (Lparen.Line > 0) indicates a parenthesized declaration. + // + // Relationship between Tok value and Specs element type: + // + // token.IMPORT *ImportSpec + // token.CONST *ValueSpec + // token.TYPE *TypeSpec + // token.VAR *ValueSpec + // + GenDecl struct { + Doc *CommentGroup // associated documentation; or nil + TokPos token.Pos // position of Tok + Tok token.Token // IMPORT, CONST, TYPE, VAR + Lparen token.Pos // position of '(', if any + Specs []Spec + Rparen token.Pos // position of ')', if any + } + + // A FuncDecl node represents a function declaration. + FuncDecl struct { + Doc *CommentGroup // associated documentation; or nil + Recv *FieldList // receiver (methods); or nil (functions) + Name *Ident // function/method name + Type *FuncType // position of Func keyword, parameters and results + Body *BlockStmt // function body; or nil (forward declaration) + } +) + + +// Pos and End implementations for declaration nodes. +// +func (d *BadDecl) Pos() token.Pos { return d.From } +func (d *GenDecl) Pos() token.Pos { return d.TokPos } +func (d *FuncDecl) Pos() token.Pos { return d.Type.Pos() } + + +func (d *BadDecl) End() token.Pos { return d.To } +func (d *GenDecl) End() token.Pos { + if d.Rparen.IsValid() { + return d.Rparen + 1 + } + return d.Specs[0].End() +} +func (d *FuncDecl) End() token.Pos { + if d.Body != nil { + return d.Body.End() + } + return d.Type.End() +} + + +// declNode() ensures that only declaration nodes can be +// assigned to a DeclNode. +// +func (d *BadDecl) declNode() {} +func (d *GenDecl) declNode() {} +func (d *FuncDecl) declNode() {} + + +// ---------------------------------------------------------------------------- +// Files and packages + +// A File node represents a Go source file. +// +// The Comments list contains all comments in the source file in order of +// appearance, including the comments that are pointed to from other nodes +// via Doc and Comment fields. +// +type File struct { + Doc *CommentGroup // associated documentation; or nil + Package token.Pos // position of "package" keyword + Name *Ident // package name + Decls []Decl // top-level declarations; or nil + Comments []*CommentGroup // list of all comments in the source file +} + + +func (f *File) Pos() token.Pos { return f.Package } +func (f *File) End() token.Pos { + if n := len(f.Decls); n > 0 { + return f.Decls[n-1].End() + } + return f.Name.End() +} + + +// A Package node represents a set of source files +// collectively building a Go package. +// +type Package struct { + Name string // package name + Scope *Scope // package scope; or nil + Files map[string]*File // Go source files by filename +} + + +func (p *Package) Pos() token.Pos { return token.NoPos } +func (p *Package) End() token.Pos { return token.NoPos } diff --git a/libgo/go/go/ast/filter.go b/libgo/go/go/ast/filter.go new file mode 100644 index 000000000..0c3cef4b2 --- /dev/null +++ b/libgo/go/go/ast/filter.go @@ -0,0 +1,429 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ast + +import "go/token" + +// ---------------------------------------------------------------------------- +// Export filtering + +func identListExports(list []*Ident) []*Ident { + j := 0 + for _, x := range list { + if x.IsExported() { + list[j] = x + j++ + } + } + return list[0:j] +} + + +// isExportedType assumes that typ is a correct type. +func isExportedType(typ Expr) bool { + switch t := typ.(type) { + case *Ident: + return t.IsExported() + case *ParenExpr: + return isExportedType(t.X) + case *SelectorExpr: + // assume t.X is a typename + return t.Sel.IsExported() + case *StarExpr: + return isExportedType(t.X) + } + return false +} + + +func fieldListExports(fields *FieldList, incomplete *bool) { + if fields == nil { + return + } + list := fields.List + j := 0 + for _, f := range list { + exported := false + if len(f.Names) == 0 { + // anonymous field + // (Note that a non-exported anonymous field + // may still refer to a type with exported + // fields, so this is not absolutely correct. + // However, this cannot be done w/o complete + // type information.) + exported = isExportedType(f.Type) + } else { + n := len(f.Names) + f.Names = identListExports(f.Names) + if len(f.Names) < n { + *incomplete = true + } + exported = len(f.Names) > 0 + } + if exported { + typeExports(f.Type) + list[j] = f + j++ + } + } + if j < len(list) { + *incomplete = true + } + fields.List = list[0:j] +} + + +func paramListExports(fields *FieldList) { + if fields == nil { + return + } + for _, f := range fields.List { + typeExports(f.Type) + } +} + + +func typeExports(typ Expr) { + switch t := typ.(type) { + case *ArrayType: + typeExports(t.Elt) + case *StructType: + fieldListExports(t.Fields, &t.Incomplete) + case *FuncType: + paramListExports(t.Params) + paramListExports(t.Results) + case *InterfaceType: + fieldListExports(t.Methods, &t.Incomplete) + case *MapType: + typeExports(t.Key) + typeExports(t.Value) + case *ChanType: + typeExports(t.Value) + } +} + + +func specExports(spec Spec) bool { + switch s := spec.(type) { + case *ValueSpec: + s.Names = identListExports(s.Names) + if len(s.Names) > 0 { + typeExports(s.Type) + return true + } + case *TypeSpec: + if s.Name.IsExported() { + typeExports(s.Type) + return true + } + } + return false +} + + +func specListExports(list []Spec) []Spec { + j := 0 + for _, s := range list { + if specExports(s) { + list[j] = s + j++ + } + } + return list[0:j] +} + + +func declExports(decl Decl) bool { + switch d := decl.(type) { + case *GenDecl: + d.Specs = specListExports(d.Specs) + return len(d.Specs) > 0 + case *FuncDecl: + d.Body = nil // strip body + return d.Name.IsExported() + } + return false +} + + +// FileExports trims the AST for a Go source file in place such that only +// exported nodes remain: all top-level identifiers which are not exported +// and their associated information (such as type, initial value, or function +// body) are removed. Non-exported fields and methods of exported types are +// stripped, and the function bodies of exported functions are set to nil. +// The File.comments list is not changed. +// +// FileExports returns true if there is an exported declaration; it returns +// false otherwise. +// +func FileExports(src *File) bool { + j := 0 + for _, d := range src.Decls { + if declExports(d) { + src.Decls[j] = d + j++ + } + } + src.Decls = src.Decls[0:j] + return j > 0 +} + + +// PackageExports trims the AST for a Go package in place such that only +// exported nodes remain. The pkg.Files list is not changed, so that file +// names and top-level package comments don't get lost. +// +// PackageExports returns true if there is an exported declaration; it +// returns false otherwise. +// +func PackageExports(pkg *Package) bool { + hasExports := false + for _, f := range pkg.Files { + if FileExports(f) { + hasExports = true + } + } + return hasExports +} + + +// ---------------------------------------------------------------------------- +// General filtering + +type Filter func(string) bool + +func filterIdentList(list []*Ident, f Filter) []*Ident { + j := 0 + for _, x := range list { + if f(x.Name) { + list[j] = x + j++ + } + } + return list[0:j] +} + + +func filterSpec(spec Spec, f Filter) bool { + switch s := spec.(type) { + case *ValueSpec: + s.Names = filterIdentList(s.Names, f) + return len(s.Names) > 0 + case *TypeSpec: + return f(s.Name.Name) + } + return false +} + + +func filterSpecList(list []Spec, f Filter) []Spec { + j := 0 + for _, s := range list { + if filterSpec(s, f) { + list[j] = s + j++ + } + } + return list[0:j] +} + + +func filterDecl(decl Decl, f Filter) bool { + switch d := decl.(type) { + case *GenDecl: + d.Specs = filterSpecList(d.Specs, f) + return len(d.Specs) > 0 + case *FuncDecl: + return f(d.Name.Name) + } + return false +} + + +// FilterFile trims the AST for a Go file in place by removing all +// names from top-level declarations (but not from parameter lists +// or inside types) that don't pass through the filter f. If the +// declaration is empty afterwards, the declaration is removed from +// the AST. +// The File.comments list is not changed. +// +// FilterFile returns true if there are any top-level declarations +// left after filtering; it returns false otherwise. +// +func FilterFile(src *File, f Filter) bool { + j := 0 + for _, d := range src.Decls { + if filterDecl(d, f) { + src.Decls[j] = d + j++ + } + } + src.Decls = src.Decls[0:j] + return j > 0 +} + + +// FilterPackage trims the AST for a Go package in place by removing all +// names from top-level declarations (but not from parameter lists +// or inside types) that don't pass through the filter f. If the +// declaration is empty afterwards, the declaration is removed from +// the AST. +// The pkg.Files list is not changed, so that file names and top-level +// package comments don't get lost. +// +// FilterPackage returns true if there are any top-level declarations +// left after filtering; it returns false otherwise. +// +func FilterPackage(pkg *Package, f Filter) bool { + hasDecls := false + for _, src := range pkg.Files { + if FilterFile(src, f) { + hasDecls = true + } + } + return hasDecls +} + + +// ---------------------------------------------------------------------------- +// Merging of package files + +// The MergeMode flags control the behavior of MergePackageFiles. +type MergeMode uint + +const ( + // If set, duplicate function declarations are excluded. + FilterFuncDuplicates MergeMode = 1 << iota + // If set, comments that are not associated with a specific + // AST node (as Doc or Comment) are excluded. + FilterUnassociatedComments +) + +// separator is an empty //-style comment that is interspersed between +// different comment groups when they are concatenated into a single group +// +var separator = &Comment{noPos, []byte("//")} + + +// MergePackageFiles creates a file AST by merging the ASTs of the +// files belonging to a package. The mode flags control merging behavior. +// +func MergePackageFiles(pkg *Package, mode MergeMode) *File { + // Count the number of package docs, comments and declarations across + // all package files. + ndocs := 0 + ncomments := 0 + ndecls := 0 + for _, f := range pkg.Files { + if f.Doc != nil { + ndocs += len(f.Doc.List) + 1 // +1 for separator + } + ncomments += len(f.Comments) + ndecls += len(f.Decls) + } + + // Collect package comments from all package files into a single + // CommentGroup - the collected package documentation. The order + // is unspecified. In general there should be only one file with + // a package comment; but it's better to collect extra comments + // than drop them on the floor. + var doc *CommentGroup + var pos token.Pos + if ndocs > 0 { + list := make([]*Comment, ndocs-1) // -1: no separator before first group + i := 0 + for _, f := range pkg.Files { + if f.Doc != nil { + if i > 0 { + // not the first group - add separator + list[i] = separator + i++ + } + for _, c := range f.Doc.List { + list[i] = c + i++ + } + if f.Package > pos { + // Keep the maximum package clause position as + // position for the package clause of the merged + // files. + pos = f.Package + } + } + } + doc = &CommentGroup{list} + } + + // Collect declarations from all package files. + var decls []Decl + if ndecls > 0 { + decls = make([]Decl, ndecls) + funcs := make(map[string]int) // map of global function name -> decls index + i := 0 // current index + n := 0 // number of filtered entries + for _, f := range pkg.Files { + for _, d := range f.Decls { + if mode&FilterFuncDuplicates != 0 { + // A language entity may be declared multiple + // times in different package files; only at + // build time declarations must be unique. + // For now, exclude multiple declarations of + // functions - keep the one with documentation. + // + // TODO(gri): Expand this filtering to other + // entities (const, type, vars) if + // multiple declarations are common. + if f, isFun := d.(*FuncDecl); isFun { + name := f.Name.Name + if j, exists := funcs[name]; exists { + // function declared already + if decls[j] != nil && decls[j].(*FuncDecl).Doc == nil { + // existing declaration has no documentation; + // ignore the existing declaration + decls[j] = nil + } else { + // ignore the new declaration + d = nil + } + n++ // filtered an entry + } else { + funcs[name] = i + } + } + } + decls[i] = d + i++ + } + } + + // Eliminate nil entries from the decls list if entries were + // filtered. We do this using a 2nd pass in order to not disturb + // the original declaration order in the source (otherwise, this + // would also invalidate the monotonically increasing position + // info within a single file). + if n > 0 { + i = 0 + for _, d := range decls { + if d != nil { + decls[i] = d + i++ + } + } + decls = decls[0:i] + } + } + + // Collect comments from all package files. + var comments []*CommentGroup + if mode&FilterUnassociatedComments == 0 { + comments = make([]*CommentGroup, ncomments) + i := 0 + for _, f := range pkg.Files { + i += copy(comments[i:], f.Comments) + } + } + + return &File{doc, pos, NewIdent(pkg.Name), decls, comments} +} diff --git a/libgo/go/go/ast/print.go b/libgo/go/go/ast/print.go new file mode 100644 index 000000000..d71490d4a --- /dev/null +++ b/libgo/go/go/ast/print.go @@ -0,0 +1,217 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file contains printing suppport for ASTs. + +package ast + +import ( + "fmt" + "go/token" + "io" + "os" + "reflect" +) + + +// A FieldFilter may be provided to Fprint to control the output. +type FieldFilter func(name string, value reflect.Value) bool + + +// NotNilFilter returns true for field values that are not nil; +// it returns false otherwise. +func NotNilFilter(_ string, value reflect.Value) bool { + v, ok := value.(interface { + IsNil() bool + }) + return !ok || !v.IsNil() +} + + +// Fprint prints the (sub-)tree starting at AST node x to w. +// +// A non-nil FieldFilter f may be provided to control the output: +// struct fields for which f(fieldname, fieldvalue) is true are +// are printed; all others are filtered from the output. +// +func Fprint(w io.Writer, x interface{}, f FieldFilter) (n int, err os.Error) { + // setup printer + p := printer{ + output: w, + filter: f, + ptrmap: make(map[interface{}]int), + last: '\n', // force printing of line number on first line + } + + // install error handler + defer func() { + n = p.written + if e := recover(); e != nil { + err = e.(localError).err // re-panics if it's not a localError + } + }() + + // print x + if x == nil { + p.printf("nil\n") + return + } + p.print(reflect.NewValue(x)) + p.printf("\n") + + return +} + + +// Print prints x to standard output, skipping nil fields. +// Print(x) is the same as Fprint(os.Stdout, x, NotNilFilter). +func Print(x interface{}) (int, os.Error) { + return Fprint(os.Stdout, x, NotNilFilter) +} + + +type printer struct { + output io.Writer + filter FieldFilter + ptrmap map[interface{}]int // *reflect.PtrValue -> line number + written int // number of bytes written to output + indent int // current indentation level + last byte // the last byte processed by Write + line int // current line number +} + + +var indent = []byte(". ") + +func (p *printer) Write(data []byte) (n int, err os.Error) { + var m int + for i, b := range data { + // invariant: data[0:n] has been written + if b == '\n' { + m, err = p.output.Write(data[n : i+1]) + n += m + if err != nil { + return + } + p.line++ + } else if p.last == '\n' { + _, err = fmt.Fprintf(p.output, "%6d ", p.line) + if err != nil { + return + } + for j := p.indent; j > 0; j-- { + _, err = p.output.Write(indent) + if err != nil { + return + } + } + } + p.last = b + } + m, err = p.output.Write(data[n:]) + n += m + return +} + + +// localError wraps locally caught os.Errors so we can distinguish +// them from genuine panics which we don't want to return as errors. +type localError struct { + err os.Error +} + + +// printf is a convenience wrapper that takes care of print errors. +func (p *printer) printf(format string, args ...interface{}) { + n, err := fmt.Fprintf(p, format, args...) + p.written += n + if err != nil { + panic(localError{err}) + } +} + + +// Implementation note: Print is written for AST nodes but could be +// used to print arbitrary data structures; such a version should +// probably be in a different package. + +func (p *printer) print(x reflect.Value) { + // Note: This test is only needed because AST nodes + // embed a token.Position, and thus all of them + // understand the String() method (but it only + // applies to the Position field). + // TODO: Should reconsider this AST design decision. + if pos, ok := x.Interface().(token.Position); ok { + p.printf("%s", pos) + return + } + + if !NotNilFilter("", x) { + p.printf("nil") + return + } + + switch v := x.(type) { + case *reflect.InterfaceValue: + p.print(v.Elem()) + + case *reflect.MapValue: + p.printf("%s (len = %d) {\n", x.Type().String(), v.Len()) + p.indent++ + for _, key := range v.Keys() { + p.print(key) + p.printf(": ") + p.print(v.Elem(key)) + } + p.indent-- + p.printf("}") + + case *reflect.PtrValue: + p.printf("*") + // type-checked ASTs may contain cycles - use ptrmap + // to keep track of objects that have been printed + // already and print the respective line number instead + ptr := v.Interface() + if line, exists := p.ptrmap[ptr]; exists { + p.printf("(obj @ %d)", line) + } else { + p.ptrmap[ptr] = p.line + p.print(v.Elem()) + } + + case *reflect.SliceValue: + if s, ok := v.Interface().([]byte); ok { + p.printf("%#q", s) + return + } + p.printf("%s (len = %d) {\n", x.Type().String(), v.Len()) + p.indent++ + for i, n := 0, v.Len(); i < n; i++ { + p.printf("%d: ", i) + p.print(v.Elem(i)) + p.printf("\n") + } + p.indent-- + p.printf("}") + + case *reflect.StructValue: + p.printf("%s {\n", x.Type().String()) + p.indent++ + t := v.Type().(*reflect.StructType) + for i, n := 0, t.NumField(); i < n; i++ { + name := t.Field(i).Name + value := v.Field(i) + if p.filter == nil || p.filter(name, value) { + p.printf("%s: ", name) + p.print(value) + p.printf("\n") + } + } + p.indent-- + p.printf("}") + + default: + p.printf("%v", x.Interface()) + } +} diff --git a/libgo/go/go/ast/scope.go b/libgo/go/go/ast/scope.go new file mode 100644 index 000000000..956a208ae --- /dev/null +++ b/libgo/go/go/ast/scope.go @@ -0,0 +1,242 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements scopes, the objects they contain, +// and object types. + +package ast + +// A Scope maintains the set of named language entities declared +// in the scope and a link to the immediately surrounding (outer) +// scope. +// +type Scope struct { + Outer *Scope + Objects []*Object // in declaration order + // Implementation note: In some cases (struct fields, + // function parameters) we need the source order of + // variables. Thus for now, we store scope entries + // in a linear list. If scopes become very large + // (say, for packages), we may need to change this + // to avoid slow lookups. +} + + +// NewScope creates a new scope nested in the outer scope. +func NewScope(outer *Scope) *Scope { + const n = 4 // initial scope capacity, must be > 0 + return &Scope{outer, make([]*Object, 0, n)} +} + + +// Lookup returns the object with the given name if it is +// found in scope s, otherwise it returns nil. Outer scopes +// are ignored. +// +// Lookup always returns nil if name is "_", even if the scope +// contains objects with that name. +// +func (s *Scope) Lookup(name string) *Object { + if name != "_" { + for _, obj := range s.Objects { + if obj.Name == name { + return obj + } + } + } + return nil +} + + +// Insert attempts to insert a named object into the scope s. +// If the scope does not contain an object with that name yet +// or if the object is named "_", Insert inserts the object +// and returns it. Otherwise, Insert leaves the scope unchanged +// and returns the object found in the scope instead. +// +func (s *Scope) Insert(obj *Object) *Object { + alt := s.Lookup(obj.Name) + if alt == nil { + s.append(obj) + alt = obj + } + return alt +} + + +func (s *Scope) append(obj *Object) { + s.Objects = append(s.Objects, obj) +} + +// ---------------------------------------------------------------------------- +// Objects + +// An Object describes a language entity such as a package, +// constant, type, variable, or function (incl. methods). +// +type Object struct { + Kind Kind + Name string // declared name + Type *Type + Decl interface{} // corresponding Field, XxxSpec or FuncDecl + N int // value of iota for this declaration +} + + +// NewObj creates a new object of a given kind and name. +func NewObj(kind Kind, name string) *Object { + return &Object{Kind: kind, Name: name} +} + + +// Kind describes what an object represents. +type Kind int + +// The list of possible Object kinds. +const ( + Bad Kind = iota // for error handling + Pkg // package + Con // constant + Typ // type + Var // variable + Fun // function or method +) + + +var objKindStrings = [...]string{ + Bad: "bad", + Pkg: "package", + Con: "const", + Typ: "type", + Var: "var", + Fun: "func", +} + + +func (kind Kind) String() string { return objKindStrings[kind] } + + +// IsExported returns whether obj is exported. +func (obj *Object) IsExported() bool { return IsExported(obj.Name) } + + +// ---------------------------------------------------------------------------- +// Types + +// A Type represents a Go type. +type Type struct { + Form Form + Obj *Object // corresponding type name, or nil + Scope *Scope // fields and methods, always present + N uint // basic type id, array length, number of function results, or channel direction + Key, Elt *Type // map key and array, pointer, slice, map or channel element + Params *Scope // function (receiver, input and result) parameters, tuple expressions (results of function calls), or nil + Expr Expr // corresponding AST expression +} + + +// NewType creates a new type of a given form. +func NewType(form Form) *Type { + return &Type{Form: form, Scope: NewScope(nil)} +} + + +// Form describes the form of a type. +type Form int + +// The list of possible type forms. +const ( + BadType Form = iota // for error handling + Unresolved // type not fully setup + Basic + Array + Struct + Pointer + Function + Method + Interface + Slice + Map + Channel + Tuple +) + + +var formStrings = [...]string{ + BadType: "badType", + Unresolved: "unresolved", + Basic: "basic", + Array: "array", + Struct: "struct", + Pointer: "pointer", + Function: "function", + Method: "method", + Interface: "interface", + Slice: "slice", + Map: "map", + Channel: "channel", + Tuple: "tuple", +} + + +func (form Form) String() string { return formStrings[form] } + + +// The list of basic type id's. +const ( + Bool = iota + Byte + Uint + Int + Float + Complex + Uintptr + String + + Uint8 + Uint16 + Uint32 + Uint64 + + Int8 + Int16 + Int32 + Int64 + + Float32 + Float64 + + Complex64 + Complex128 + + // TODO(gri) ideal types are missing +) + + +var BasicTypes = map[uint]string{ + Bool: "bool", + Byte: "byte", + Uint: "uint", + Int: "int", + Float: "float", + Complex: "complex", + Uintptr: "uintptr", + String: "string", + + Uint8: "uint8", + Uint16: "uint16", + Uint32: "uint32", + Uint64: "uint64", + + Int8: "int8", + Int16: "int16", + Int32: "int32", + Int64: "int64", + + Float32: "float32", + Float64: "float64", + + Complex64: "complex64", + Complex128: "complex128", +} diff --git a/libgo/go/go/ast/walk.go b/libgo/go/go/ast/walk.go new file mode 100644 index 000000000..875a92f3f --- /dev/null +++ b/libgo/go/go/ast/walk.go @@ -0,0 +1,396 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ast + +import "fmt" + +// A Visitor's Visit method is invoked for each node encountered by Walk. +// If the result visitor w is not nil, Walk visits each of the children +// of node with the visitor w, followed by a call of w.Visit(nil). +type Visitor interface { + Visit(node Node) (w Visitor) +} + + +// Helper functions for common node lists. They may be empty. + +func walkIdentList(v Visitor, list []*Ident) { + for _, x := range list { + Walk(v, x) + } +} + + +func walkExprList(v Visitor, list []Expr) { + for _, x := range list { + Walk(v, x) + } +} + + +func walkStmtList(v Visitor, list []Stmt) { + for _, x := range list { + Walk(v, x) + } +} + + +func walkDeclList(v Visitor, list []Decl) { + for _, x := range list { + Walk(v, x) + } +} + + +// TODO(gri): Investigate if providing a closure to Walk leads to +// simpler use (and may help eliminate Inspect in turn). + +// Walk traverses an AST in depth-first order: It starts by calling +// v.Visit(node); node must not be nil. If the visitor w returned by +// v.Visit(node) is not nil, Walk is invoked recursively with visitor +// w for each of the non-nil children of node, followed by a call of +// w.Visit(nil). +// +func Walk(v Visitor, node Node) { + if v = v.Visit(node); v == nil { + return + } + + // walk children + // (the order of the cases matches the order + // of the corresponding node types in ast.go) + switch n := node.(type) { + // Comments and fields + case *Comment: + // nothing to do + + case *CommentGroup: + for _, c := range n.List { + Walk(v, c) + } + + case *Field: + if n.Doc != nil { + Walk(v, n.Doc) + } + walkIdentList(v, n.Names) + Walk(v, n.Type) + if n.Tag != nil { + Walk(v, n.Tag) + } + if n.Comment != nil { + Walk(v, n.Comment) + } + + case *FieldList: + for _, f := range n.List { + Walk(v, f) + } + + // Expressions + case *BadExpr, *Ident, *BasicLit: + // nothing to do + + case *Ellipsis: + if n.Elt != nil { + Walk(v, n.Elt) + } + + case *FuncLit: + Walk(v, n.Type) + Walk(v, n.Body) + + case *CompositeLit: + if n.Type != nil { + Walk(v, n.Type) + } + walkExprList(v, n.Elts) + + case *ParenExpr: + Walk(v, n.X) + + case *SelectorExpr: + Walk(v, n.X) + Walk(v, n.Sel) + + case *IndexExpr: + Walk(v, n.X) + Walk(v, n.Index) + + case *SliceExpr: + Walk(v, n.X) + if n.Low != nil { + Walk(v, n.Low) + } + if n.High != nil { + Walk(v, n.High) + } + + case *TypeAssertExpr: + Walk(v, n.X) + if n.Type != nil { + Walk(v, n.Type) + } + + case *CallExpr: + Walk(v, n.Fun) + walkExprList(v, n.Args) + + case *StarExpr: + Walk(v, n.X) + + case *UnaryExpr: + Walk(v, n.X) + + case *BinaryExpr: + Walk(v, n.X) + Walk(v, n.Y) + + case *KeyValueExpr: + Walk(v, n.Key) + Walk(v, n.Value) + + // Types + case *ArrayType: + if n.Len != nil { + Walk(v, n.Len) + } + Walk(v, n.Elt) + + case *StructType: + Walk(v, n.Fields) + + case *FuncType: + Walk(v, n.Params) + if n.Results != nil { + Walk(v, n.Results) + } + + case *InterfaceType: + Walk(v, n.Methods) + + case *MapType: + Walk(v, n.Key) + Walk(v, n.Value) + + case *ChanType: + Walk(v, n.Value) + + // Statements + case *BadStmt: + // nothing to do + + case *DeclStmt: + Walk(v, n.Decl) + + case *EmptyStmt: + // nothing to do + + case *LabeledStmt: + Walk(v, n.Label) + Walk(v, n.Stmt) + + case *ExprStmt: + Walk(v, n.X) + + case *IncDecStmt: + Walk(v, n.X) + + case *AssignStmt: + walkExprList(v, n.Lhs) + walkExprList(v, n.Rhs) + + case *GoStmt: + Walk(v, n.Call) + + case *DeferStmt: + Walk(v, n.Call) + + case *ReturnStmt: + walkExprList(v, n.Results) + + case *BranchStmt: + if n.Label != nil { + Walk(v, n.Label) + } + + case *BlockStmt: + walkStmtList(v, n.List) + + case *IfStmt: + if n.Init != nil { + Walk(v, n.Init) + } + if n.Cond != nil { + Walk(v, n.Cond) + } + Walk(v, n.Body) + if n.Else != nil { + Walk(v, n.Else) + } + + case *CaseClause: + walkExprList(v, n.Values) + walkStmtList(v, n.Body) + + case *SwitchStmt: + if n.Init != nil { + Walk(v, n.Init) + } + if n.Tag != nil { + Walk(v, n.Tag) + } + Walk(v, n.Body) + + case *TypeCaseClause: + for _, x := range n.Types { + Walk(v, x) + } + walkStmtList(v, n.Body) + + case *TypeSwitchStmt: + if n.Init != nil { + Walk(v, n.Init) + } + Walk(v, n.Assign) + Walk(v, n.Body) + + case *CommClause: + if n.Lhs != nil { + Walk(v, n.Lhs) + } + if n.Rhs != nil { + Walk(v, n.Rhs) + } + walkStmtList(v, n.Body) + + case *SelectStmt: + Walk(v, n.Body) + + case *ForStmt: + if n.Init != nil { + Walk(v, n.Init) + } + if n.Cond != nil { + Walk(v, n.Cond) + } + if n.Post != nil { + Walk(v, n.Post) + } + Walk(v, n.Body) + + case *RangeStmt: + Walk(v, n.Key) + if n.Value != nil { + Walk(v, n.Value) + } + Walk(v, n.X) + Walk(v, n.Body) + + // Declarations + case *ImportSpec: + if n.Doc != nil { + Walk(v, n.Doc) + } + if n.Name != nil { + Walk(v, n.Name) + } + Walk(v, n.Path) + if n.Comment != nil { + Walk(v, n.Comment) + } + + case *ValueSpec: + if n.Doc != nil { + Walk(v, n.Doc) + } + walkIdentList(v, n.Names) + if n.Type != nil { + Walk(v, n.Type) + } + walkExprList(v, n.Values) + if n.Comment != nil { + Walk(v, n.Comment) + } + + case *TypeSpec: + if n.Doc != nil { + Walk(v, n.Doc) + } + Walk(v, n.Name) + Walk(v, n.Type) + if n.Comment != nil { + Walk(v, n.Comment) + } + + case *BadDecl: + // nothing to do + + case *GenDecl: + if n.Doc != nil { + Walk(v, n.Doc) + } + for _, s := range n.Specs { + Walk(v, s) + } + + case *FuncDecl: + if n.Doc != nil { + Walk(v, n.Doc) + } + if n.Recv != nil { + Walk(v, n.Recv) + } + Walk(v, n.Name) + Walk(v, n.Type) + if n.Body != nil { + Walk(v, n.Body) + } + + // Files and packages + case *File: + if n.Doc != nil { + Walk(v, n.Doc) + } + Walk(v, n.Name) + walkDeclList(v, n.Decls) + for _, g := range n.Comments { + Walk(v, g) + } + // don't walk n.Comments - they have been + // visited already through the individual + // nodes + + case *Package: + for _, f := range n.Files { + Walk(v, f) + } + + default: + fmt.Printf("ast.Walk: unexpected node type %T", n) + panic("ast.Walk") + } + + v.Visit(nil) +} + + +type inspector func(Node) bool + +func (f inspector) Visit(node Node) Visitor { + if f(node) { + return f + } + return nil +} + + +// Inspect traverses an AST in depth-first order: It starts by calling +// f(node); node must not be nil. If f returns true, Inspect invokes f +// for all the non-nil children of node, recursively. +// +func Inspect(node Node, f func(Node) bool) { + Walk(inspector(f), node) +} diff --git a/libgo/go/go/doc/comment.go b/libgo/go/go/doc/comment.go new file mode 100644 index 000000000..9ff0bd536 --- /dev/null +++ b/libgo/go/go/doc/comment.go @@ -0,0 +1,357 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Godoc comment extraction and comment -> HTML formatting. + +package doc + +import ( + "go/ast" + "io" + "regexp" + "strings" + "template" // for htmlEscape +) + + +func isWhitespace(ch byte) bool { return ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r' } + + +func stripTrailingWhitespace(s string) string { + i := len(s) + for i > 0 && isWhitespace(s[i-1]) { + i-- + } + return s[0:i] +} + + +// CommentText returns the text of comment, +// with the comment markers - //, /*, and */ - removed. +func CommentText(comment *ast.CommentGroup) string { + if comment == nil { + return "" + } + comments := make([]string, len(comment.List)) + for i, c := range comment.List { + comments[i] = string(c.Text) + } + + lines := make([]string, 0, 10) // most comments are less than 10 lines + for _, c := range comments { + // Remove comment markers. + // The parser has given us exactly the comment text. + switch c[1] { + case '/': + //-style comment + c = c[2:] + // Remove leading space after //, if there is one. + // TODO(gri) This appears to be necessary in isolated + // cases (bignum.RatFromString) - why? + if len(c) > 0 && c[0] == ' ' { + c = c[1:] + } + case '*': + /*-style comment */ + c = c[2 : len(c)-2] + } + + // Split on newlines. + cl := strings.Split(c, "\n", -1) + + // Walk lines, stripping trailing white space and adding to list. + for _, l := range cl { + lines = append(lines, stripTrailingWhitespace(l)) + } + } + + // Remove leading blank lines; convert runs of + // interior blank lines to a single blank line. + n := 0 + for _, line := range lines { + if line != "" || n > 0 && lines[n-1] != "" { + lines[n] = line + n++ + } + } + lines = lines[0:n] + + // Add final "" entry to get trailing newline from Join. + if n > 0 && lines[n-1] != "" { + lines = append(lines, "") + } + + return strings.Join(lines, "\n") +} + + +// Split bytes into lines. +func split(text []byte) [][]byte { + // count lines + n := 0 + last := 0 + for i, c := range text { + if c == '\n' { + last = i + 1 + n++ + } + } + if last < len(text) { + n++ + } + + // split + out := make([][]byte, n) + last = 0 + n = 0 + for i, c := range text { + if c == '\n' { + out[n] = text[last : i+1] + last = i + 1 + n++ + } + } + if last < len(text) { + out[n] = text[last:] + } + + return out +} + + +var ( + ldquo = []byte("“") + rdquo = []byte("”") +) + +// Escape comment text for HTML. If nice is set, +// also turn `` into “ and '' into ”. +func commentEscape(w io.Writer, s []byte, nice bool) { + last := 0 + if nice { + for i := 0; i < len(s)-1; i++ { + ch := s[i] + if ch == s[i+1] && (ch == '`' || ch == '\'') { + template.HTMLEscape(w, s[last:i]) + last = i + 2 + switch ch { + case '`': + w.Write(ldquo) + case '\'': + w.Write(rdquo) + } + i++ // loop will add one more + } + } + } + template.HTMLEscape(w, s[last:]) +} + + +const ( + // Regexp for Go identifiers + identRx = `[a-zA-Z_][a-zA-Z_0-9]*` // TODO(gri) ASCII only for now - fix this + + // Regexp for URLs + protocol = `(https?|ftp|file|gopher|mailto|news|nntp|telnet|wais|prospero):` + hostPart = `[a-zA-Z0-9_@\-]+` + filePart = `[a-zA-Z0-9_?%#~&/\-+=]+` + urlRx = protocol + `//` + // http:// + hostPart + `([.:]` + hostPart + `)*/?` + // //www.google.com:8080/ + filePart + `([:.,]` + filePart + `)*` +) + +var matchRx = regexp.MustCompile(`(` + identRx + `)|(` + urlRx + `)`) + +var ( + html_a = []byte(``) + html_enda = []byte("") + html_i = []byte("") + html_endi = []byte("") + html_p = []byte("

\n") + html_endp = []byte("

\n") + html_pre = []byte("
")
+	html_endpre = []byte("
\n") +) + + +// Emphasize and escape a line of text for HTML. URLs are converted into links; +// if the URL also appears in the words map, the link is taken from the map (if +// the corresponding map value is the empty string, the URL is not converted +// into a link). Go identifiers that appear in the words map are italicized; if +// the corresponding map value is not the empty string, it is considered a URL +// and the word is converted into a link. If nice is set, the remaining text's +// appearance is improved where it makes sense (e.g., `` is turned into “ +// and '' into ”). +func emphasize(w io.Writer, line []byte, words map[string]string, nice bool) { + for { + m := matchRx.FindSubmatchIndex(line) + if m == nil { + break + } + // m >= 6 (two parenthesized sub-regexps in matchRx, 1st one is identRx) + + // write text before match + commentEscape(w, line[0:m[0]], nice) + + // analyze match + match := line[m[0]:m[1]] + url := "" + italics := false + if words != nil { + url, italics = words[string(match)] + } + if m[2] < 0 { + // didn't match against first parenthesized sub-regexp; must be match against urlRx + if !italics { + // no alternative URL in words list, use match instead + url = string(match) + } + italics = false // don't italicize URLs + } + + // write match + if len(url) > 0 { + w.Write(html_a) + template.HTMLEscape(w, []byte(url)) + w.Write(html_aq) + } + if italics { + w.Write(html_i) + } + commentEscape(w, match, nice) + if italics { + w.Write(html_endi) + } + if len(url) > 0 { + w.Write(html_enda) + } + + // advance + line = line[m[1]:] + } + commentEscape(w, line, nice) +} + + +func indentLen(s []byte) int { + i := 0 + for i < len(s) && (s[i] == ' ' || s[i] == '\t') { + i++ + } + return i +} + + +func isBlank(s []byte) bool { return len(s) == 0 || (len(s) == 1 && s[0] == '\n') } + + +func commonPrefix(a, b []byte) []byte { + i := 0 + for i < len(a) && i < len(b) && a[i] == b[i] { + i++ + } + return a[0:i] +} + + +func unindent(block [][]byte) { + if len(block) == 0 { + return + } + + // compute maximum common white prefix + prefix := block[0][0:indentLen(block[0])] + for _, line := range block { + if !isBlank(line) { + prefix = commonPrefix(prefix, line[0:indentLen(line)]) + } + } + n := len(prefix) + + // remove + for i, line := range block { + if !isBlank(line) { + block[i] = line[n:] + } + } +} + + +// Convert comment text to formatted HTML. +// The comment was prepared by DocReader, +// so it is known not to have leading, trailing blank lines +// nor to have trailing spaces at the end of lines. +// The comment markers have already been removed. +// +// Turn each run of multiple \n into

+// Turn each run of indented lines into a 

 block without indent.
+//
+// URLs in the comment text are converted into links; if the URL also appears
+// in the words map, the link is taken from the map (if the corresponding map
+// value is the empty string, the URL is not converted into a link).
+//
+// Go identifiers that appear in the words map are italicized; if the corresponding
+// map value is not the empty string, it is considered a URL and the word is converted
+// into a link.
+func ToHTML(w io.Writer, s []byte, words map[string]string) {
+	inpara := false
+
+	close := func() {
+		if inpara {
+			w.Write(html_endp)
+			inpara = false
+		}
+	}
+	open := func() {
+		if !inpara {
+			w.Write(html_p)
+			inpara = true
+		}
+	}
+
+	lines := split(s)
+	unindent(lines)
+	for i := 0; i < len(lines); {
+		line := lines[i]
+		if isBlank(line) {
+			// close paragraph
+			close()
+			i++
+			continue
+		}
+		if indentLen(line) > 0 {
+			// close paragraph
+			close()
+
+			// count indented or blank lines
+			j := i + 1
+			for j < len(lines) && (isBlank(lines[j]) || indentLen(lines[j]) > 0) {
+				j++
+			}
+			// but not trailing blank lines
+			for j > i && isBlank(lines[j-1]) {
+				j--
+			}
+			block := lines[i:j]
+			i = j
+
+			unindent(block)
+
+			// put those lines in a pre block
+			w.Write(html_pre)
+			for _, line := range block {
+				emphasize(w, line, nil, false) // no nice text formatting
+			}
+			w.Write(html_endpre)
+			continue
+		}
+		// open paragraph
+		open()
+		emphasize(w, lines[i], words, true) // nice text formatting
+		i++
+	}
+	close()
+}
diff --git a/libgo/go/go/doc/doc.go b/libgo/go/go/doc/doc.go
new file mode 100644
index 000000000..e46857cb8
--- /dev/null
+++ b/libgo/go/go/doc/doc.go
@@ -0,0 +1,650 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// The doc package extracts source code documentation from a Go AST.
+package doc
+
+import (
+	"go/ast"
+	"go/token"
+	"regexp"
+	"sort"
+)
+
+
+// ----------------------------------------------------------------------------
+
+type typeDoc struct {
+	// len(decl.Specs) == 1, and the element type is *ast.TypeSpec
+	// if the type declaration hasn't been seen yet, decl is nil
+	decl *ast.GenDecl
+	// values, factory functions, and methods associated with the type
+	values    []*ast.GenDecl // consts and vars
+	factories map[string]*ast.FuncDecl
+	methods   map[string]*ast.FuncDecl
+}
+
+
+// docReader accumulates documentation for a single package.
+// It modifies the AST: Comments (declaration documentation)
+// that have been collected by the DocReader are set to nil
+// in the respective AST nodes so that they are not printed
+// twice (once when printing the documentation and once when
+// printing the corresponding AST node).
+//
+type docReader struct {
+	doc     *ast.CommentGroup // package documentation, if any
+	pkgName string
+	values  []*ast.GenDecl // consts and vars
+	types   map[string]*typeDoc
+	funcs   map[string]*ast.FuncDecl
+	bugs    []*ast.CommentGroup
+}
+
+
+func (doc *docReader) init(pkgName string) {
+	doc.pkgName = pkgName
+	doc.types = make(map[string]*typeDoc)
+	doc.funcs = make(map[string]*ast.FuncDecl)
+}
+
+
+func (doc *docReader) addDoc(comments *ast.CommentGroup) {
+	if doc.doc == nil {
+		// common case: just one package comment
+		doc.doc = comments
+		return
+	}
+
+	// More than one package comment: Usually there will be only
+	// one file with a package comment, but it's better to collect
+	// all comments than drop them on the floor.
+	// (This code isn't particularly clever - no amortized doubling is
+	// used - but this situation occurs rarely and is not time-critical.)
+	n1 := len(doc.doc.List)
+	n2 := len(comments.List)
+	list := make([]*ast.Comment, n1+1+n2) // + 1 for separator line
+	copy(list, doc.doc.List)
+	list[n1] = &ast.Comment{token.NoPos, []byte("//")} // separator line
+	copy(list[n1+1:], comments.List)
+	doc.doc = &ast.CommentGroup{list}
+}
+
+
+func (doc *docReader) addType(decl *ast.GenDecl) {
+	spec := decl.Specs[0].(*ast.TypeSpec)
+	typ := doc.lookupTypeDoc(spec.Name.Name)
+	// typ should always be != nil since declared types
+	// are always named - be conservative and check
+	if typ != nil {
+		// a type should be added at most once, so typ.decl
+		// should be nil - if it isn't, simply overwrite it
+		typ.decl = decl
+	}
+}
+
+
+func (doc *docReader) lookupTypeDoc(name string) *typeDoc {
+	if name == "" {
+		return nil // no type docs for anonymous types
+	}
+	if tdoc, found := doc.types[name]; found {
+		return tdoc
+	}
+	// type wasn't found - add one without declaration
+	tdoc := &typeDoc{nil, nil, make(map[string]*ast.FuncDecl), make(map[string]*ast.FuncDecl)}
+	doc.types[name] = tdoc
+	return tdoc
+}
+
+
+func baseTypeName(typ ast.Expr) string {
+	switch t := typ.(type) {
+	case *ast.Ident:
+		// if the type is not exported, the effect to
+		// a client is as if there were no type name
+		if t.IsExported() {
+			return string(t.Name)
+		}
+	case *ast.StarExpr:
+		return baseTypeName(t.X)
+	}
+	return ""
+}
+
+
+func (doc *docReader) addValue(decl *ast.GenDecl) {
+	// determine if decl should be associated with a type
+	// Heuristic: For each typed entry, determine the type name, if any.
+	//            If there is exactly one type name that is sufficiently
+	//            frequent, associate the decl with the respective type.
+	domName := ""
+	domFreq := 0
+	prev := ""
+	for _, s := range decl.Specs {
+		if v, ok := s.(*ast.ValueSpec); ok {
+			name := ""
+			switch {
+			case v.Type != nil:
+				// a type is present; determine its name
+				name = baseTypeName(v.Type)
+			case decl.Tok == token.CONST:
+				// no type is present but we have a constant declaration;
+				// use the previous type name (w/o more type information
+				// we cannot handle the case of unnamed variables with
+				// initializer expressions except for some trivial cases)
+				name = prev
+			}
+			if name != "" {
+				// entry has a named type
+				if domName != "" && domName != name {
+					// more than one type name - do not associate
+					// with any type
+					domName = ""
+					break
+				}
+				domName = name
+				domFreq++
+			}
+			prev = name
+		}
+	}
+
+	// determine values list
+	const threshold = 0.75
+	values := &doc.values
+	if domName != "" && domFreq >= int(float64(len(decl.Specs))*threshold) {
+		// typed entries are sufficiently frequent
+		typ := doc.lookupTypeDoc(domName)
+		if typ != nil {
+			values = &typ.values // associate with that type
+		}
+	}
+
+	*values = append(*values, decl)
+}
+
+
+// Helper function to set the table entry for function f. Makes sure that
+// at least one f with associated documentation is stored in table, if there
+// are multiple f's with the same name.
+func setFunc(table map[string]*ast.FuncDecl, f *ast.FuncDecl) {
+	name := f.Name.Name
+	if g, exists := table[name]; exists && g.Doc != nil {
+		// a function with the same name has already been registered;
+		// since it has documentation, assume f is simply another
+		// implementation and ignore it
+		// TODO(gri) consider collecting all functions, or at least
+		//           all comments
+		return
+	}
+	// function doesn't exist or has no documentation; use f
+	table[name] = f
+}
+
+
+func (doc *docReader) addFunc(fun *ast.FuncDecl) {
+	name := fun.Name.Name
+
+	// determine if it should be associated with a type
+	if fun.Recv != nil {
+		// method
+		typ := doc.lookupTypeDoc(baseTypeName(fun.Recv.List[0].Type))
+		if typ != nil {
+			// exported receiver type
+			setFunc(typ.methods, fun)
+		}
+		// otherwise don't show the method
+		// TODO(gri): There may be exported methods of non-exported types
+		// that can be called because of exported values (consts, vars, or
+		// function results) of that type. Could determine if that is the
+		// case and then show those methods in an appropriate section.
+		return
+	}
+
+	// perhaps a factory function
+	// determine result type, if any
+	if fun.Type.Results.NumFields() >= 1 {
+		res := fun.Type.Results.List[0]
+		if len(res.Names) <= 1 {
+			// exactly one (named or anonymous) result associated
+			// with the first type in result signature (there may
+			// be more than one result)
+			tname := baseTypeName(res.Type)
+			typ := doc.lookupTypeDoc(tname)
+			if typ != nil {
+				// named and exported result type
+
+				// Work-around for failure of heuristic: In package os
+				// too many functions are considered factory functions
+				// for the Error type. Eliminate manually for now as
+				// this appears to be the only important case in the
+				// current library where the heuristic fails.
+				if doc.pkgName == "os" && tname == "Error" &&
+					name != "NewError" && name != "NewSyscallError" {
+					// not a factory function for os.Error
+					setFunc(doc.funcs, fun) // treat as ordinary function
+					return
+				}
+
+				setFunc(typ.factories, fun)
+				return
+			}
+		}
+	}
+
+	// ordinary function
+	setFunc(doc.funcs, fun)
+}
+
+
+func (doc *docReader) addDecl(decl ast.Decl) {
+	switch d := decl.(type) {
+	case *ast.GenDecl:
+		if len(d.Specs) > 0 {
+			switch d.Tok {
+			case token.CONST, token.VAR:
+				// constants and variables are always handled as a group
+				doc.addValue(d)
+			case token.TYPE:
+				// types are handled individually
+				for _, spec := range d.Specs {
+					// make a (fake) GenDecl node for this TypeSpec
+					// (we need to do this here - as opposed to just
+					// for printing - so we don't lose the GenDecl
+					// documentation)
+					//
+					// TODO(gri): Consider just collecting the TypeSpec
+					// node (and copy in the GenDecl.doc if there is no
+					// doc in the TypeSpec - this is currently done in
+					// makeTypeDocs below). Simpler data structures, but
+					// would lose GenDecl documentation if the TypeSpec
+					// has documentation as well.
+					doc.addType(&ast.GenDecl{d.Doc, d.Pos(), token.TYPE, token.NoPos, []ast.Spec{spec}, token.NoPos})
+					// A new GenDecl node is created, no need to nil out d.Doc.
+				}
+			}
+		}
+	case *ast.FuncDecl:
+		doc.addFunc(d)
+	}
+}
+
+
+func copyCommentList(list []*ast.Comment) []*ast.Comment {
+	return append([]*ast.Comment(nil), list...)
+}
+
+var (
+	bug_markers = regexp.MustCompile("^/[/*][ \t]*BUG\\(.*\\):[ \t]*") // BUG(uid):
+	bug_content = regexp.MustCompile("[^ \n\r\t]+")                    // at least one non-whitespace char
+)
+
+
+// addFile adds the AST for a source file to the docReader.
+// Adding the same AST multiple times is a no-op.
+//
+func (doc *docReader) addFile(src *ast.File) {
+	// add package documentation
+	if src.Doc != nil {
+		doc.addDoc(src.Doc)
+		src.Doc = nil // doc consumed - remove from ast.File node
+	}
+
+	// add all declarations
+	for _, decl := range src.Decls {
+		doc.addDecl(decl)
+	}
+
+	// collect BUG(...) comments
+	for _, c := range src.Comments {
+		text := c.List[0].Text
+		if m := bug_markers.FindIndex(text); m != nil {
+			// found a BUG comment; maybe empty
+			if btxt := text[m[1]:]; bug_content.Match(btxt) {
+				// non-empty BUG comment; collect comment without BUG prefix
+				list := copyCommentList(c.List)
+				list[0].Text = text[m[1]:]
+				doc.bugs = append(doc.bugs, &ast.CommentGroup{list})
+			}
+		}
+	}
+	src.Comments = nil // consumed unassociated comments - remove from ast.File node
+}
+
+
+func NewFileDoc(file *ast.File) *PackageDoc {
+	var r docReader
+	r.init(file.Name.Name)
+	r.addFile(file)
+	return r.newDoc("", nil)
+}
+
+
+func NewPackageDoc(pkg *ast.Package, importpath string) *PackageDoc {
+	var r docReader
+	r.init(pkg.Name)
+	filenames := make([]string, len(pkg.Files))
+	i := 0
+	for filename, f := range pkg.Files {
+		r.addFile(f)
+		filenames[i] = filename
+		i++
+	}
+	return r.newDoc(importpath, filenames)
+}
+
+
+// ----------------------------------------------------------------------------
+// Conversion to external representation
+
+// ValueDoc is the documentation for a group of declared
+// values, either vars or consts.
+//
+type ValueDoc struct {
+	Doc   string
+	Decl  *ast.GenDecl
+	order int
+}
+
+type sortValueDoc []*ValueDoc
+
+func (p sortValueDoc) Len() int      { return len(p) }
+func (p sortValueDoc) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
+
+
+func declName(d *ast.GenDecl) string {
+	if len(d.Specs) != 1 {
+		return ""
+	}
+
+	switch v := d.Specs[0].(type) {
+	case *ast.ValueSpec:
+		return v.Names[0].Name
+	case *ast.TypeSpec:
+		return v.Name.Name
+	}
+
+	return ""
+}
+
+
+func (p sortValueDoc) Less(i, j int) bool {
+	// sort by name
+	// pull blocks (name = "") up to top
+	// in original order
+	if ni, nj := declName(p[i].Decl), declName(p[j].Decl); ni != nj {
+		return ni < nj
+	}
+	return p[i].order < p[j].order
+}
+
+
+func makeValueDocs(list []*ast.GenDecl, tok token.Token) []*ValueDoc {
+	d := make([]*ValueDoc, len(list)) // big enough in any case
+	n := 0
+	for i, decl := range list {
+		if decl.Tok == tok {
+			d[n] = &ValueDoc{CommentText(decl.Doc), decl, i}
+			n++
+			decl.Doc = nil // doc consumed - removed from AST
+		}
+	}
+	d = d[0:n]
+	sort.Sort(sortValueDoc(d))
+	return d
+}
+
+
+// FuncDoc is the documentation for a func declaration,
+// either a top-level function or a method function.
+//
+type FuncDoc struct {
+	Doc  string
+	Recv ast.Expr // TODO(rsc): Would like string here
+	Name string
+	Decl *ast.FuncDecl
+}
+
+type sortFuncDoc []*FuncDoc
+
+func (p sortFuncDoc) Len() int           { return len(p) }
+func (p sortFuncDoc) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
+func (p sortFuncDoc) Less(i, j int) bool { return p[i].Name < p[j].Name }
+
+
+func makeFuncDocs(m map[string]*ast.FuncDecl) []*FuncDoc {
+	d := make([]*FuncDoc, len(m))
+	i := 0
+	for _, f := range m {
+		doc := new(FuncDoc)
+		doc.Doc = CommentText(f.Doc)
+		f.Doc = nil // doc consumed - remove from ast.FuncDecl node
+		if f.Recv != nil {
+			doc.Recv = f.Recv.List[0].Type
+		}
+		doc.Name = f.Name.Name
+		doc.Decl = f
+		d[i] = doc
+		i++
+	}
+	sort.Sort(sortFuncDoc(d))
+	return d
+}
+
+
+// TypeDoc is the documentation for a declared type.
+// Consts and Vars are sorted lists of constants and variables of (mostly) that type.
+// Factories is a sorted list of factory functions that return that type.
+// Methods is a sorted list of method functions on that type.
+type TypeDoc struct {
+	Doc       string
+	Type      *ast.TypeSpec
+	Consts    []*ValueDoc
+	Vars      []*ValueDoc
+	Factories []*FuncDoc
+	Methods   []*FuncDoc
+	Decl      *ast.GenDecl
+	order     int
+}
+
+type sortTypeDoc []*TypeDoc
+
+func (p sortTypeDoc) Len() int      { return len(p) }
+func (p sortTypeDoc) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
+func (p sortTypeDoc) Less(i, j int) bool {
+	// sort by name
+	// pull blocks (name = "") up to top
+	// in original order
+	if ni, nj := p[i].Type.Name.Name, p[j].Type.Name.Name; ni != nj {
+		return ni < nj
+	}
+	return p[i].order < p[j].order
+}
+
+
+// NOTE(rsc): This would appear not to be correct for type ( )
+// blocks, but the doc extractor above has split them into
+// individual declarations.
+func (doc *docReader) makeTypeDocs(m map[string]*typeDoc) []*TypeDoc {
+	d := make([]*TypeDoc, len(m))
+	i := 0
+	for _, old := range m {
+		// all typeDocs should have a declaration associated with
+		// them after processing an entire package - be conservative
+		// and check
+		if decl := old.decl; decl != nil {
+			typespec := decl.Specs[0].(*ast.TypeSpec)
+			t := new(TypeDoc)
+			doc := typespec.Doc
+			typespec.Doc = nil // doc consumed - remove from ast.TypeSpec node
+			if doc == nil {
+				// no doc associated with the spec, use the declaration doc, if any
+				doc = decl.Doc
+			}
+			decl.Doc = nil // doc consumed - remove from ast.Decl node
+			t.Doc = CommentText(doc)
+			t.Type = typespec
+			t.Consts = makeValueDocs(old.values, token.CONST)
+			t.Vars = makeValueDocs(old.values, token.VAR)
+			t.Factories = makeFuncDocs(old.factories)
+			t.Methods = makeFuncDocs(old.methods)
+			t.Decl = old.decl
+			t.order = i
+			d[i] = t
+			i++
+		} else {
+			// no corresponding type declaration found - move any associated
+			// values, factory functions, and methods back to the top-level
+			// so that they are not lost (this should only happen if a package
+			// file containing the explicit type declaration is missing or if
+			// an unqualified type name was used after a "." import)
+			// 1) move values
+			doc.values = append(doc.values, old.values...)
+			// 2) move factory functions
+			for name, f := range old.factories {
+				doc.funcs[name] = f
+			}
+			// 3) move methods
+			for name, f := range old.methods {
+				// don't overwrite functions with the same name
+				if _, found := doc.funcs[name]; !found {
+					doc.funcs[name] = f
+				}
+			}
+		}
+	}
+	d = d[0:i] // some types may have been ignored
+	sort.Sort(sortTypeDoc(d))
+	return d
+}
+
+
+func makeBugDocs(list []*ast.CommentGroup) []string {
+	d := make([]string, len(list))
+	for i, g := range list {
+		d[i] = CommentText(g)
+	}
+	return d
+}
+
+
+// PackageDoc is the documentation for an entire package.
+//
+type PackageDoc struct {
+	PackageName string
+	ImportPath  string
+	Filenames   []string
+	Doc         string
+	Consts      []*ValueDoc
+	Types       []*TypeDoc
+	Vars        []*ValueDoc
+	Funcs       []*FuncDoc
+	Bugs        []string
+}
+
+
+// newDoc returns the accumulated documentation for the package.
+//
+func (doc *docReader) newDoc(importpath string, filenames []string) *PackageDoc {
+	p := new(PackageDoc)
+	p.PackageName = doc.pkgName
+	p.ImportPath = importpath
+	sort.SortStrings(filenames)
+	p.Filenames = filenames
+	p.Doc = CommentText(doc.doc)
+	// makeTypeDocs may extend the list of doc.values and
+	// doc.funcs and thus must be called before any other
+	// function consuming those lists
+	p.Types = doc.makeTypeDocs(doc.types)
+	p.Consts = makeValueDocs(doc.values, token.CONST)
+	p.Vars = makeValueDocs(doc.values, token.VAR)
+	p.Funcs = makeFuncDocs(doc.funcs)
+	p.Bugs = makeBugDocs(doc.bugs)
+	return p
+}
+
+
+// ----------------------------------------------------------------------------
+// Filtering by name
+
+type Filter func(string) bool
+
+
+func matchDecl(d *ast.GenDecl, f Filter) bool {
+	for _, d := range d.Specs {
+		switch v := d.(type) {
+		case *ast.ValueSpec:
+			for _, name := range v.Names {
+				if f(name.Name) {
+					return true
+				}
+			}
+		case *ast.TypeSpec:
+			if f(v.Name.Name) {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+
+func filterValueDocs(a []*ValueDoc, f Filter) []*ValueDoc {
+	w := 0
+	for _, vd := range a {
+		if matchDecl(vd.Decl, f) {
+			a[w] = vd
+			w++
+		}
+	}
+	return a[0:w]
+}
+
+
+func filterFuncDocs(a []*FuncDoc, f Filter) []*FuncDoc {
+	w := 0
+	for _, fd := range a {
+		if f(fd.Name) {
+			a[w] = fd
+			w++
+		}
+	}
+	return a[0:w]
+}
+
+
+func filterTypeDocs(a []*TypeDoc, f Filter) []*TypeDoc {
+	w := 0
+	for _, td := range a {
+		n := 0 // number of matches
+		if matchDecl(td.Decl, f) {
+			n = 1
+		} else {
+			// type name doesn't match, but we may have matching consts, vars, factories or methods
+			td.Consts = filterValueDocs(td.Consts, f)
+			td.Vars = filterValueDocs(td.Vars, f)
+			td.Factories = filterFuncDocs(td.Factories, f)
+			td.Methods = filterFuncDocs(td.Methods, f)
+			n += len(td.Consts) + len(td.Vars) + len(td.Factories) + len(td.Methods)
+		}
+		if n > 0 {
+			a[w] = td
+			w++
+		}
+	}
+	return a[0:w]
+}
+
+
+// Filter eliminates documentation for names that don't pass through the filter f.
+// TODO: Recognize "Type.Method" as a name.
+//
+func (p *PackageDoc) Filter(f Filter) {
+	p.Consts = filterValueDocs(p.Consts, f)
+	p.Vars = filterValueDocs(p.Vars, f)
+	p.Types = filterTypeDocs(p.Types, f)
+	p.Funcs = filterFuncDocs(p.Funcs, f)
+	p.Doc = "" // don't show top-level package doc
+}
diff --git a/libgo/go/go/parser/interface.go b/libgo/go/go/parser/interface.go
new file mode 100644
index 000000000..84d699a67
--- /dev/null
+++ b/libgo/go/go/parser/interface.go
@@ -0,0 +1,208 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This file contains the exported entry points for invoking the parser.
+
+package parser
+
+import (
+	"bytes"
+	"go/ast"
+	"go/scanner"
+	"go/token"
+	"io"
+	"io/ioutil"
+	"os"
+	pathutil "path"
+)
+
+
+// If src != nil, readSource converts src to a []byte if possible;
+// otherwise it returns an error. If src == nil, readSource returns
+// the result of reading the file specified by filename.
+//
+func readSource(filename string, src interface{}) ([]byte, os.Error) {
+	if src != nil {
+		switch s := src.(type) {
+		case string:
+			return []byte(s), nil
+		case []byte:
+			return s, nil
+		case *bytes.Buffer:
+			// is io.Reader, but src is already available in []byte form
+			if s != nil {
+				return s.Bytes(), nil
+			}
+		case io.Reader:
+			var buf bytes.Buffer
+			_, err := io.Copy(&buf, s)
+			if err != nil {
+				return nil, err
+			}
+			return buf.Bytes(), nil
+		default:
+			return nil, os.ErrorString("invalid source")
+		}
+	}
+
+	return ioutil.ReadFile(filename)
+}
+
+
+func (p *parser) parseEOF() os.Error {
+	p.expect(token.EOF)
+	return p.GetError(scanner.Sorted)
+}
+
+
+// ParseExpr parses a Go expression and returns the corresponding
+// AST node. The fset, filename, and src arguments have the same interpretation
+// as for ParseFile. If there is an error, the result expression
+// may be nil or contain a partial AST.
+//
+func ParseExpr(fset *token.FileSet, filename string, src interface{}) (ast.Expr, os.Error) {
+	data, err := readSource(filename, src)
+	if err != nil {
+		return nil, err
+	}
+
+	var p parser
+	p.init(fset, filename, data, 0)
+	x := p.parseExpr()
+	if p.tok == token.SEMICOLON {
+		p.next() // consume automatically inserted semicolon, if any
+	}
+	return x, p.parseEOF()
+}
+
+
+// ParseStmtList parses a list of Go statements and returns the list
+// of corresponding AST nodes. The fset, filename, and src arguments have the same
+// interpretation as for ParseFile. If there is an error, the node
+// list may be nil or contain partial ASTs.
+//
+func ParseStmtList(fset *token.FileSet, filename string, src interface{}) ([]ast.Stmt, os.Error) {
+	data, err := readSource(filename, src)
+	if err != nil {
+		return nil, err
+	}
+
+	var p parser
+	p.init(fset, filename, data, 0)
+	return p.parseStmtList(), p.parseEOF()
+}
+
+
+// ParseDeclList parses a list of Go declarations and returns the list
+// of corresponding AST nodes. The fset, filename, and src arguments have the same
+// interpretation as for ParseFile. If there is an error, the node
+// list may be nil or contain partial ASTs.
+//
+func ParseDeclList(fset *token.FileSet, filename string, src interface{}) ([]ast.Decl, os.Error) {
+	data, err := readSource(filename, src)
+	if err != nil {
+		return nil, err
+	}
+
+	var p parser
+	p.init(fset, filename, data, 0)
+	return p.parseDeclList(), p.parseEOF()
+}
+
+
+// ParseFile parses the source code of a single Go source file and returns
+// the corresponding ast.File node. The source code may be provided via
+// the filename of the source file, or via the src parameter.
+//
+// If src != nil, ParseFile parses the source from src and the filename is
+// only used when recording position information. The type of the argument
+// for the src parameter must be string, []byte, or io.Reader.
+//
+// If src == nil, ParseFile parses the file specified by filename.
+//
+// The mode parameter controls the amount of source text parsed and other
+// optional parser functionality. Position information is recorded in the
+// file set fset.
+//
+// If the source couldn't be read, the returned AST is nil and the error
+// indicates the specific failure. If the source was read but syntax
+// errors were found, the result is a partial AST (with ast.BadX nodes
+// representing the fragments of erroneous source code). Multiple errors
+// are returned via a scanner.ErrorList which is sorted by file position.
+//
+func ParseFile(fset *token.FileSet, filename string, src interface{}, mode uint) (*ast.File, os.Error) {
+	data, err := readSource(filename, src)
+	if err != nil {
+		return nil, err
+	}
+
+	var p parser
+	p.init(fset, filename, data, mode)
+	return p.parseFile(), p.GetError(scanner.NoMultiples) // parseFile() reads to EOF
+}
+
+
+// ParseFiles calls ParseFile for each file in the filenames list and returns
+// a map of package name -> package AST with all the packages found. The mode
+// bits are passed to ParseFile unchanged. Position information is recorded
+// in the file set fset.
+//
+// Files with parse errors are ignored. In this case the map of packages may
+// be incomplete (missing packages and/or incomplete packages) and the first
+// error encountered is returned.
+//
+func ParseFiles(fset *token.FileSet, filenames []string, mode uint) (pkgs map[string]*ast.Package, first os.Error) {
+	pkgs = make(map[string]*ast.Package)
+	for _, filename := range filenames {
+		if src, err := ParseFile(fset, filename, nil, mode); err == nil {
+			name := src.Name.Name
+			pkg, found := pkgs[name]
+			if !found {
+				pkg = &ast.Package{name, nil, make(map[string]*ast.File)}
+				pkgs[name] = pkg
+			}
+			pkg.Files[filename] = src
+		} else if first == nil {
+			first = err
+		}
+	}
+	return
+}
+
+
+// ParseDir calls ParseFile for the files in the directory specified by path and
+// returns a map of package name -> package AST with all the packages found. If
+// filter != nil, only the files with os.FileInfo entries passing through the filter
+// are considered. The mode bits are passed to ParseFile unchanged. Position
+// information is recorded in the file set fset.
+//
+// If the directory couldn't be read, a nil map and the respective error are
+// returned. If a parse error occurred, a non-nil but incomplete map and the
+// error are returned.
+//
+func ParseDir(fset *token.FileSet, path string, filter func(*os.FileInfo) bool, mode uint) (map[string]*ast.Package, os.Error) {
+	fd, err := os.Open(path, os.O_RDONLY, 0)
+	if err != nil {
+		return nil, err
+	}
+	defer fd.Close()
+
+	list, err := fd.Readdir(-1)
+	if err != nil {
+		return nil, err
+	}
+
+	filenames := make([]string, len(list))
+	n := 0
+	for i := 0; i < len(list); i++ {
+		d := &list[i]
+		if filter == nil || filter(d) {
+			filenames[n] = pathutil.Join(path, d.Name)
+			n++
+		}
+	}
+	filenames = filenames[0:n]
+
+	return ParseFiles(fset, filenames, mode)
+}
diff --git a/libgo/go/go/parser/parser.go b/libgo/go/go/parser/parser.go
new file mode 100644
index 000000000..f1746e040
--- /dev/null
+++ b/libgo/go/go/parser/parser.go
@@ -0,0 +1,1885 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// A parser for Go source files. Input may be provided in a variety of
+// forms (see the various Parse* functions); the output is an abstract
+// syntax tree (AST) representing the Go source. The parser is invoked
+// through one of the Parse* functions.
+//
+package parser
+
+import (
+	"fmt"
+	"go/ast"
+	"go/scanner"
+	"go/token"
+)
+
+
+// noPos is used when there is no corresponding source position for a token.
+var noPos token.Position
+
+
+// The mode parameter to the Parse* functions is a set of flags (or 0).
+// They control the amount of source code parsed and other optional
+// parser functionality.
+//
+const (
+	PackageClauseOnly uint = 1 << iota // parsing stops after package clause
+	ImportsOnly                        // parsing stops after import declarations
+	ParseComments                      // parse comments and add them to AST
+	Trace                              // print a trace of parsed productions
+)
+
+
+// The parser structure holds the parser's internal state.
+type parser struct {
+	file *token.File
+	scanner.ErrorVector
+	scanner scanner.Scanner
+
+	// Tracing/debugging
+	mode   uint // parsing mode
+	trace  bool // == (mode & Trace != 0)
+	indent uint // indentation used for tracing output
+
+	// Comments
+	comments    []*ast.CommentGroup
+	leadComment *ast.CommentGroup // the last lead comment
+	lineComment *ast.CommentGroup // the last line comment
+
+	// Next token
+	pos token.Pos   // token position
+	tok token.Token // one token look-ahead
+	lit []byte      // token literal
+
+	// Non-syntactic parser control
+	exprLev int // < 0: in control clause, >= 0: in expression
+}
+
+
+// scannerMode returns the scanner mode bits given the parser's mode bits.
+func scannerMode(mode uint) uint {
+	var m uint = scanner.InsertSemis
+	if mode&ParseComments != 0 {
+		m |= scanner.ScanComments
+	}
+	return m
+}
+
+
+func (p *parser) init(fset *token.FileSet, filename string, src []byte, mode uint) {
+	p.file = fset.AddFile(filename, fset.Base(), len(src))
+	p.scanner.Init(p.file, src, p, scannerMode(mode))
+	p.mode = mode
+	p.trace = mode&Trace != 0 // for convenience (p.trace is used frequently)
+	p.next()
+}
+
+
+// ----------------------------------------------------------------------------
+// Parsing support
+
+func (p *parser) printTrace(a ...interface{}) {
+	const dots = ". . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . " +
+		". . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . "
+	const n = uint(len(dots))
+	pos := p.file.Position(p.pos)
+	fmt.Printf("%5d:%3d: ", pos.Line, pos.Column)
+	i := 2 * p.indent
+	for ; i > n; i -= n {
+		fmt.Print(dots)
+	}
+	fmt.Print(dots[0:i])
+	fmt.Println(a...)
+}
+
+
+func trace(p *parser, msg string) *parser {
+	p.printTrace(msg, "(")
+	p.indent++
+	return p
+}
+
+
+// Usage pattern: defer un(trace(p, "..."));
+func un(p *parser) {
+	p.indent--
+	p.printTrace(")")
+}
+
+
+// Advance to the next token.
+func (p *parser) next0() {
+	// Because of one-token look-ahead, print the previous token
+	// when tracing as it provides a more readable output. The
+	// very first token (!p.pos.IsValid()) is not initialized
+	// (it is token.ILLEGAL), so don't print it .
+	if p.trace && p.pos.IsValid() {
+		s := p.tok.String()
+		switch {
+		case p.tok.IsLiteral():
+			p.printTrace(s, string(p.lit))
+		case p.tok.IsOperator(), p.tok.IsKeyword():
+			p.printTrace("\"" + s + "\"")
+		default:
+			p.printTrace(s)
+		}
+	}
+
+	p.pos, p.tok, p.lit = p.scanner.Scan()
+}
+
+// Consume a comment and return it and the line on which it ends.
+func (p *parser) consumeComment() (comment *ast.Comment, endline int) {
+	// /*-style comments may end on a different line than where they start.
+	// Scan the comment for '\n' chars and adjust endline accordingly.
+	endline = p.file.Line(p.pos)
+	if p.lit[1] == '*' {
+		for _, b := range p.lit {
+			if b == '\n' {
+				endline++
+			}
+		}
+	}
+
+	comment = &ast.Comment{p.pos, p.lit}
+	p.next0()
+
+	return
+}
+
+
+// Consume a group of adjacent comments, add it to the parser's
+// comments list, and return it together with the line at which
+// the last comment in the group ends. An empty line or non-comment
+// token terminates a comment group.
+//
+func (p *parser) consumeCommentGroup() (comments *ast.CommentGroup, endline int) {
+	var list []*ast.Comment
+	endline = p.file.Line(p.pos)
+	for p.tok == token.COMMENT && endline+1 >= p.file.Line(p.pos) {
+		var comment *ast.Comment
+		comment, endline = p.consumeComment()
+		list = append(list, comment)
+	}
+
+	// add comment group to the comments list
+	comments = &ast.CommentGroup{list}
+	p.comments = append(p.comments, comments)
+
+	return
+}
+
+
+// Advance to the next non-comment token. In the process, collect
+// any comment groups encountered, and remember the last lead and
+// and line comments.
+//
+// A lead comment is a comment group that starts and ends in a
+// line without any other tokens and that is followed by a non-comment
+// token on the line immediately after the comment group.
+//
+// A line comment is a comment group that follows a non-comment
+// token on the same line, and that has no tokens after it on the line
+// where it ends.
+//
+// Lead and line comments may be considered documentation that is
+// stored in the AST.
+//
+func (p *parser) next() {
+	p.leadComment = nil
+	p.lineComment = nil
+	line := p.file.Line(p.pos) // current line
+	p.next0()
+
+	if p.tok == token.COMMENT {
+		var comment *ast.CommentGroup
+		var endline int
+
+		if p.file.Line(p.pos) == line {
+			// The comment is on same line as previous token; it
+			// cannot be a lead comment but may be a line comment.
+			comment, endline = p.consumeCommentGroup()
+			if p.file.Line(p.pos) != endline {
+				// The next token is on a different line, thus
+				// the last comment group is a line comment.
+				p.lineComment = comment
+			}
+		}
+
+		// consume successor comments, if any
+		endline = -1
+		for p.tok == token.COMMENT {
+			comment, endline = p.consumeCommentGroup()
+		}
+
+		if endline+1 == p.file.Line(p.pos) {
+			// The next token is following on the line immediately after the
+			// comment group, thus the last comment group is a lead comment.
+			p.leadComment = comment
+		}
+	}
+}
+
+
+func (p *parser) error(pos token.Pos, msg string) {
+	p.Error(p.file.Position(pos), msg)
+}
+
+
+func (p *parser) errorExpected(pos token.Pos, msg string) {
+	msg = "expected " + msg
+	if pos == p.pos {
+		// the error happened at the current position;
+		// make the error message more specific
+		if p.tok == token.SEMICOLON && p.lit[0] == '\n' {
+			msg += ", found newline"
+		} else {
+			msg += ", found '" + p.tok.String() + "'"
+			if p.tok.IsLiteral() {
+				msg += " " + string(p.lit)
+			}
+		}
+	}
+	p.error(pos, msg)
+}
+
+
+func (p *parser) expect(tok token.Token) token.Pos {
+	pos := p.pos
+	if p.tok != tok {
+		p.errorExpected(pos, "'"+tok.String()+"'")
+	}
+	p.next() // make progress
+	return pos
+}
+
+
+func (p *parser) expectSemi() {
+	if p.tok != token.RPAREN && p.tok != token.RBRACE {
+		p.expect(token.SEMICOLON)
+	}
+}
+
+
+// ----------------------------------------------------------------------------
+// Identifiers
+
+func (p *parser) parseIdent() *ast.Ident {
+	pos := p.pos
+	name := "_"
+	if p.tok == token.IDENT {
+		name = string(p.lit)
+		p.next()
+	} else {
+		p.expect(token.IDENT) // use expect() error handling
+	}
+	return &ast.Ident{pos, name, nil}
+}
+
+
+func (p *parser) parseIdentList() (list []*ast.Ident) {
+	if p.trace {
+		defer un(trace(p, "IdentList"))
+	}
+
+	list = append(list, p.parseIdent())
+	for p.tok == token.COMMA {
+		p.next()
+		list = append(list, p.parseIdent())
+	}
+
+	return
+}
+
+
+// ----------------------------------------------------------------------------
+// Common productions
+
+func (p *parser) parseExprList() (list []ast.Expr) {
+	if p.trace {
+		defer un(trace(p, "ExpressionList"))
+	}
+
+	list = append(list, p.parseExpr())
+	for p.tok == token.COMMA {
+		p.next()
+		list = append(list, p.parseExpr())
+	}
+
+	return
+}
+
+
+// ----------------------------------------------------------------------------
+// Types
+
+func (p *parser) parseType() ast.Expr {
+	if p.trace {
+		defer un(trace(p, "Type"))
+	}
+
+	typ := p.tryType()
+
+	if typ == nil {
+		pos := p.pos
+		p.errorExpected(pos, "type")
+		p.next() // make progress
+		return &ast.BadExpr{pos, p.pos}
+	}
+
+	return typ
+}
+
+
+func (p *parser) parseQualifiedIdent() ast.Expr {
+	if p.trace {
+		defer un(trace(p, "QualifiedIdent"))
+	}
+
+	var x ast.Expr = p.parseIdent()
+	if p.tok == token.PERIOD {
+		// first identifier is a package identifier
+		p.next()
+		sel := p.parseIdent()
+		x = &ast.SelectorExpr{x, sel}
+	}
+	return x
+}
+
+
+func (p *parser) parseTypeName() ast.Expr {
+	if p.trace {
+		defer un(trace(p, "TypeName"))
+	}
+
+	return p.parseQualifiedIdent()
+}
+
+
+func (p *parser) parseArrayType(ellipsisOk bool) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "ArrayType"))
+	}
+
+	lbrack := p.expect(token.LBRACK)
+	var len ast.Expr
+	if ellipsisOk && p.tok == token.ELLIPSIS {
+		len = &ast.Ellipsis{p.pos, nil}
+		p.next()
+	} else if p.tok != token.RBRACK {
+		len = p.parseExpr()
+	}
+	p.expect(token.RBRACK)
+	elt := p.parseType()
+
+	return &ast.ArrayType{lbrack, len, elt}
+}
+
+
+func (p *parser) makeIdentList(list []ast.Expr) []*ast.Ident {
+	idents := make([]*ast.Ident, len(list))
+	for i, x := range list {
+		ident, isIdent := x.(*ast.Ident)
+		if !isIdent {
+			pos := x.(ast.Expr).Pos()
+			p.errorExpected(pos, "identifier")
+			ident = &ast.Ident{pos, "_", nil}
+		}
+		idents[i] = ident
+	}
+	return idents
+}
+
+
+func (p *parser) parseFieldDecl() *ast.Field {
+	if p.trace {
+		defer un(trace(p, "FieldDecl"))
+	}
+
+	doc := p.leadComment
+
+	// fields
+	list, typ := p.parseVarList(false)
+
+	// optional tag
+	var tag *ast.BasicLit
+	if p.tok == token.STRING {
+		tag = &ast.BasicLit{p.pos, p.tok, p.lit}
+		p.next()
+	}
+
+	// analyze case
+	var idents []*ast.Ident
+	if typ != nil {
+		// IdentifierList Type
+		idents = p.makeIdentList(list)
+	} else {
+		// ["*"] TypeName (AnonymousField)
+		typ = list[0] // we always have at least one element
+		if n := len(list); n > 1 || !isTypeName(deref(typ)) {
+			pos := typ.Pos()
+			p.errorExpected(pos, "anonymous field")
+			typ = &ast.BadExpr{pos, list[n-1].End()}
+		}
+	}
+
+	p.expectSemi()
+
+	return &ast.Field{doc, idents, typ, tag, p.lineComment}
+}
+
+
+func (p *parser) parseStructType() *ast.StructType {
+	if p.trace {
+		defer un(trace(p, "StructType"))
+	}
+
+	pos := p.expect(token.STRUCT)
+	lbrace := p.expect(token.LBRACE)
+	var list []*ast.Field
+	for p.tok == token.IDENT || p.tok == token.MUL || p.tok == token.LPAREN {
+		// a field declaration cannot start with a '(' but we accept
+		// it here for more robust parsing and better error messages
+		// (parseFieldDecl will check and complain if necessary)
+		list = append(list, p.parseFieldDecl())
+	}
+	rbrace := p.expect(token.RBRACE)
+
+	return &ast.StructType{pos, &ast.FieldList{lbrace, list, rbrace}, false}
+}
+
+
+func (p *parser) parsePointerType() *ast.StarExpr {
+	if p.trace {
+		defer un(trace(p, "PointerType"))
+	}
+
+	star := p.expect(token.MUL)
+	base := p.parseType()
+
+	return &ast.StarExpr{star, base}
+}
+
+
+func (p *parser) tryVarType(isParam bool) ast.Expr {
+	if isParam && p.tok == token.ELLIPSIS {
+		pos := p.pos
+		p.next()
+		typ := p.tryType() // don't use parseType so we can provide better error message
+		if typ == nil {
+			p.error(pos, "'...' parameter is missing type")
+			typ = &ast.BadExpr{pos, p.pos}
+		}
+		if p.tok != token.RPAREN {
+			p.error(pos, "can use '...' with last parameter type only")
+		}
+		return &ast.Ellipsis{pos, typ}
+	}
+	return p.tryType()
+}
+
+
+func (p *parser) parseVarType(isParam bool) ast.Expr {
+	typ := p.tryVarType(isParam)
+	if typ == nil {
+		pos := p.pos
+		p.errorExpected(pos, "type")
+		p.next() // make progress
+		typ = &ast.BadExpr{pos, p.pos}
+	}
+	return typ
+}
+
+
+func (p *parser) parseVarList(isParam bool) (list []ast.Expr, typ ast.Expr) {
+	if p.trace {
+		defer un(trace(p, "VarList"))
+	}
+
+	// a list of identifiers looks like a list of type names
+	for {
+		// parseVarType accepts any type (including parenthesized ones)
+		// even though the syntax does not permit them here: we
+		// accept them all for more robust parsing and complain
+		// afterwards
+		list = append(list, p.parseVarType(isParam))
+		if p.tok != token.COMMA {
+			break
+		}
+		p.next()
+	}
+
+	// if we had a list of identifiers, it must be followed by a type
+	typ = p.tryVarType(isParam)
+
+	return
+}
+
+
+func (p *parser) parseParameterList(ellipsisOk bool) (params []*ast.Field) {
+	if p.trace {
+		defer un(trace(p, "ParameterList"))
+	}
+
+	list, typ := p.parseVarList(ellipsisOk)
+	if typ != nil {
+		// IdentifierList Type
+		idents := p.makeIdentList(list)
+		params = append(params, &ast.Field{nil, idents, typ, nil, nil})
+		if p.tok == token.COMMA {
+			p.next()
+		}
+
+		for p.tok != token.RPAREN && p.tok != token.EOF {
+			idents := p.parseIdentList()
+			typ := p.parseVarType(ellipsisOk)
+			params = append(params, &ast.Field{nil, idents, typ, nil, nil})
+			if p.tok != token.COMMA {
+				break
+			}
+			p.next()
+		}
+
+	} else {
+		// Type { "," Type } (anonymous parameters)
+		params = make([]*ast.Field, len(list))
+		for i, x := range list {
+			params[i] = &ast.Field{Type: x}
+		}
+	}
+
+	return
+}
+
+
+func (p *parser) parseParameters(ellipsisOk bool) *ast.FieldList {
+	if p.trace {
+		defer un(trace(p, "Parameters"))
+	}
+
+	var params []*ast.Field
+	lparen := p.expect(token.LPAREN)
+	if p.tok != token.RPAREN {
+		params = p.parseParameterList(ellipsisOk)
+	}
+	rparen := p.expect(token.RPAREN)
+
+	return &ast.FieldList{lparen, params, rparen}
+}
+
+
+func (p *parser) parseResult() *ast.FieldList {
+	if p.trace {
+		defer un(trace(p, "Result"))
+	}
+
+	if p.tok == token.LPAREN {
+		return p.parseParameters(false)
+	}
+
+	typ := p.tryType()
+	if typ != nil {
+		list := make([]*ast.Field, 1)
+		list[0] = &ast.Field{Type: typ}
+		return &ast.FieldList{List: list}
+	}
+
+	return nil
+}
+
+
+func (p *parser) parseSignature() (params, results *ast.FieldList) {
+	if p.trace {
+		defer un(trace(p, "Signature"))
+	}
+
+	params = p.parseParameters(true)
+	results = p.parseResult()
+
+	return
+}
+
+
+func (p *parser) parseFuncType() *ast.FuncType {
+	if p.trace {
+		defer un(trace(p, "FuncType"))
+	}
+
+	pos := p.expect(token.FUNC)
+	params, results := p.parseSignature()
+
+	return &ast.FuncType{pos, params, results}
+}
+
+
+func (p *parser) parseMethodSpec() *ast.Field {
+	if p.trace {
+		defer un(trace(p, "MethodSpec"))
+	}
+
+	doc := p.leadComment
+	var idents []*ast.Ident
+	var typ ast.Expr
+	x := p.parseQualifiedIdent()
+	if ident, isIdent := x.(*ast.Ident); isIdent && p.tok == token.LPAREN {
+		// method
+		idents = []*ast.Ident{ident}
+		params, results := p.parseSignature()
+		typ = &ast.FuncType{token.NoPos, params, results}
+	} else {
+		// embedded interface
+		typ = x
+	}
+	p.expectSemi()
+
+	return &ast.Field{doc, idents, typ, nil, p.lineComment}
+}
+
+
+func (p *parser) parseInterfaceType() *ast.InterfaceType {
+	if p.trace {
+		defer un(trace(p, "InterfaceType"))
+	}
+
+	pos := p.expect(token.INTERFACE)
+	lbrace := p.expect(token.LBRACE)
+	var list []*ast.Field
+	for p.tok == token.IDENT {
+		list = append(list, p.parseMethodSpec())
+	}
+	rbrace := p.expect(token.RBRACE)
+
+	return &ast.InterfaceType{pos, &ast.FieldList{lbrace, list, rbrace}, false}
+}
+
+
+func (p *parser) parseMapType() *ast.MapType {
+	if p.trace {
+		defer un(trace(p, "MapType"))
+	}
+
+	pos := p.expect(token.MAP)
+	p.expect(token.LBRACK)
+	key := p.parseType()
+	p.expect(token.RBRACK)
+	value := p.parseType()
+
+	return &ast.MapType{pos, key, value}
+}
+
+
+func (p *parser) parseChanType() *ast.ChanType {
+	if p.trace {
+		defer un(trace(p, "ChanType"))
+	}
+
+	pos := p.pos
+	dir := ast.SEND | ast.RECV
+	if p.tok == token.CHAN {
+		p.next()
+		if p.tok == token.ARROW {
+			p.next()
+			dir = ast.SEND
+		}
+	} else {
+		p.expect(token.ARROW)
+		p.expect(token.CHAN)
+		dir = ast.RECV
+	}
+	value := p.parseType()
+
+	return &ast.ChanType{pos, dir, value}
+}
+
+
+func (p *parser) tryRawType(ellipsisOk bool) ast.Expr {
+	switch p.tok {
+	case token.IDENT:
+		return p.parseTypeName()
+	case token.LBRACK:
+		return p.parseArrayType(ellipsisOk)
+	case token.STRUCT:
+		return p.parseStructType()
+	case token.MUL:
+		return p.parsePointerType()
+	case token.FUNC:
+		return p.parseFuncType()
+	case token.INTERFACE:
+		return p.parseInterfaceType()
+	case token.MAP:
+		return p.parseMapType()
+	case token.CHAN, token.ARROW:
+		return p.parseChanType()
+	case token.LPAREN:
+		lparen := p.pos
+		p.next()
+		typ := p.parseType()
+		rparen := p.expect(token.RPAREN)
+		return &ast.ParenExpr{lparen, typ, rparen}
+	}
+
+	// no type found
+	return nil
+}
+
+
+func (p *parser) tryType() ast.Expr { return p.tryRawType(false) }
+
+
+// ----------------------------------------------------------------------------
+// Blocks
+
+func (p *parser) parseStmtList() (list []ast.Stmt) {
+	if p.trace {
+		defer un(trace(p, "StatementList"))
+	}
+
+	for p.tok != token.CASE && p.tok != token.DEFAULT && p.tok != token.RBRACE && p.tok != token.EOF {
+		list = append(list, p.parseStmt())
+	}
+
+	return
+}
+
+
+func (p *parser) parseBody() *ast.BlockStmt {
+	if p.trace {
+		defer un(trace(p, "Body"))
+	}
+
+	lbrace := p.expect(token.LBRACE)
+	list := p.parseStmtList()
+	rbrace := p.expect(token.RBRACE)
+
+	return &ast.BlockStmt{lbrace, list, rbrace}
+}
+
+
+func (p *parser) parseBlockStmt() *ast.BlockStmt {
+	if p.trace {
+		defer un(trace(p, "BlockStmt"))
+	}
+
+	lbrace := p.expect(token.LBRACE)
+	list := p.parseStmtList()
+	rbrace := p.expect(token.RBRACE)
+
+	return &ast.BlockStmt{lbrace, list, rbrace}
+}
+
+
+// ----------------------------------------------------------------------------
+// Expressions
+
+func (p *parser) parseFuncTypeOrLit() ast.Expr {
+	if p.trace {
+		defer un(trace(p, "FuncTypeOrLit"))
+	}
+
+	typ := p.parseFuncType()
+	if p.tok != token.LBRACE {
+		// function type only
+		return typ
+	}
+
+	p.exprLev++
+	body := p.parseBody()
+	p.exprLev--
+
+	return &ast.FuncLit{typ, body}
+}
+
+
+// parseOperand may return an expression or a raw type (incl. array
+// types of the form [...]T. Callers must verify the result.
+//
+func (p *parser) parseOperand() ast.Expr {
+	if p.trace {
+		defer un(trace(p, "Operand"))
+	}
+
+	switch p.tok {
+	case token.IDENT:
+		return p.parseIdent()
+
+	case token.INT, token.FLOAT, token.IMAG, token.CHAR, token.STRING:
+		x := &ast.BasicLit{p.pos, p.tok, p.lit}
+		p.next()
+		return x
+
+	case token.LPAREN:
+		lparen := p.pos
+		p.next()
+		p.exprLev++
+		x := p.parseExpr()
+		p.exprLev--
+		rparen := p.expect(token.RPAREN)
+		return &ast.ParenExpr{lparen, x, rparen}
+
+	case token.FUNC:
+		return p.parseFuncTypeOrLit()
+
+	default:
+		t := p.tryRawType(true) // could be type for composite literal or conversion
+		if t != nil {
+			return t
+		}
+	}
+
+	pos := p.pos
+	p.errorExpected(pos, "operand")
+	p.next() // make progress
+	return &ast.BadExpr{pos, p.pos}
+}
+
+
+func (p *parser) parseSelectorOrTypeAssertion(x ast.Expr) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "SelectorOrTypeAssertion"))
+	}
+
+	p.expect(token.PERIOD)
+	if p.tok == token.IDENT {
+		// selector
+		sel := p.parseIdent()
+		return &ast.SelectorExpr{x, sel}
+	}
+
+	// type assertion
+	p.expect(token.LPAREN)
+	var typ ast.Expr
+	if p.tok == token.TYPE {
+		// type switch: typ == nil
+		p.next()
+	} else {
+		typ = p.parseType()
+	}
+	p.expect(token.RPAREN)
+
+	return &ast.TypeAssertExpr{x, typ}
+}
+
+
+func (p *parser) parseIndexOrSlice(x ast.Expr) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "IndexOrSlice"))
+	}
+
+	lbrack := p.expect(token.LBRACK)
+	p.exprLev++
+	var low, high ast.Expr
+	isSlice := false
+	if p.tok != token.COLON {
+		low = p.parseExpr()
+	}
+	if p.tok == token.COLON {
+		isSlice = true
+		p.next()
+		if p.tok != token.RBRACK {
+			high = p.parseExpr()
+		}
+	}
+	p.exprLev--
+	rbrack := p.expect(token.RBRACK)
+
+	if isSlice {
+		return &ast.SliceExpr{x, lbrack, low, high, rbrack}
+	}
+	return &ast.IndexExpr{x, lbrack, low, rbrack}
+}
+
+
+func (p *parser) parseCallOrConversion(fun ast.Expr) *ast.CallExpr {
+	if p.trace {
+		defer un(trace(p, "CallOrConversion"))
+	}
+
+	lparen := p.expect(token.LPAREN)
+	p.exprLev++
+	var list []ast.Expr
+	var ellipsis token.Pos
+	for p.tok != token.RPAREN && p.tok != token.EOF && !ellipsis.IsValid() {
+		list = append(list, p.parseExpr())
+		if p.tok == token.ELLIPSIS {
+			ellipsis = p.pos
+			p.next()
+		}
+		if p.tok != token.COMMA {
+			break
+		}
+		p.next()
+	}
+	p.exprLev--
+	rparen := p.expect(token.RPAREN)
+
+	return &ast.CallExpr{fun, lparen, list, ellipsis, rparen}
+}
+
+
+func (p *parser) parseElement(keyOk bool) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "Element"))
+	}
+
+	if p.tok == token.LBRACE {
+		return p.parseLiteralValue(nil)
+	}
+
+	x := p.parseExpr()
+	if keyOk && p.tok == token.COLON {
+		colon := p.pos
+		p.next()
+		x = &ast.KeyValueExpr{x, colon, p.parseElement(false)}
+	}
+	return x
+}
+
+
+func (p *parser) parseElementList() (list []ast.Expr) {
+	if p.trace {
+		defer un(trace(p, "ElementList"))
+	}
+
+	for p.tok != token.RBRACE && p.tok != token.EOF {
+		list = append(list, p.parseElement(true))
+		if p.tok != token.COMMA {
+			break
+		}
+		p.next()
+	}
+
+	return
+}
+
+
+func (p *parser) parseLiteralValue(typ ast.Expr) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "LiteralValue"))
+	}
+
+	lbrace := p.expect(token.LBRACE)
+	var elts []ast.Expr
+	p.exprLev++
+	if p.tok != token.RBRACE {
+		elts = p.parseElementList()
+	}
+	p.exprLev--
+	rbrace := p.expect(token.RBRACE)
+	return &ast.CompositeLit{typ, lbrace, elts, rbrace}
+}
+
+
+// checkExpr checks that x is an expression (and not a type).
+func (p *parser) checkExpr(x ast.Expr) ast.Expr {
+	switch t := unparen(x).(type) {
+	case *ast.BadExpr:
+	case *ast.Ident:
+	case *ast.BasicLit:
+	case *ast.FuncLit:
+	case *ast.CompositeLit:
+	case *ast.ParenExpr:
+		panic("unreachable")
+	case *ast.SelectorExpr:
+	case *ast.IndexExpr:
+	case *ast.SliceExpr:
+	case *ast.TypeAssertExpr:
+		if t.Type == nil {
+			// the form X.(type) is only allowed in type switch expressions
+			p.errorExpected(x.Pos(), "expression")
+			x = &ast.BadExpr{x.Pos(), x.End()}
+		}
+	case *ast.CallExpr:
+	case *ast.StarExpr:
+	case *ast.UnaryExpr:
+		if t.Op == token.RANGE {
+			// the range operator is only allowed at the top of a for statement
+			p.errorExpected(x.Pos(), "expression")
+			x = &ast.BadExpr{x.Pos(), x.End()}
+		}
+	case *ast.BinaryExpr:
+	default:
+		// all other nodes are not proper expressions
+		p.errorExpected(x.Pos(), "expression")
+		x = &ast.BadExpr{x.Pos(), x.End()}
+	}
+	return x
+}
+
+
+// isTypeName returns true iff x is a (qualified) TypeName.
+func isTypeName(x ast.Expr) bool {
+	switch t := x.(type) {
+	case *ast.BadExpr:
+	case *ast.Ident:
+	case *ast.SelectorExpr:
+		_, isIdent := t.X.(*ast.Ident)
+		return isIdent
+	default:
+		return false // all other nodes are not type names
+	}
+	return true
+}
+
+
+// isLiteralType returns true iff x is a legal composite literal type.
+func isLiteralType(x ast.Expr) bool {
+	switch t := x.(type) {
+	case *ast.BadExpr:
+	case *ast.Ident:
+	case *ast.SelectorExpr:
+		_, isIdent := t.X.(*ast.Ident)
+		return isIdent
+	case *ast.ArrayType:
+	case *ast.StructType:
+	case *ast.MapType:
+	default:
+		return false // all other nodes are not legal composite literal types
+	}
+	return true
+}
+
+
+// If x is of the form *T, deref returns T, otherwise it returns x.
+func deref(x ast.Expr) ast.Expr {
+	if p, isPtr := x.(*ast.StarExpr); isPtr {
+		x = p.X
+	}
+	return x
+}
+
+
+// If x is of the form (T), unparen returns unparen(T), otherwise it returns x.
+func unparen(x ast.Expr) ast.Expr {
+	if p, isParen := x.(*ast.ParenExpr); isParen {
+		x = unparen(p.X)
+	}
+	return x
+}
+
+
+// checkExprOrType checks that x is an expression or a type
+// (and not a raw type such as [...]T).
+//
+func (p *parser) checkExprOrType(x ast.Expr) ast.Expr {
+	switch t := unparen(x).(type) {
+	case *ast.ParenExpr:
+		panic("unreachable")
+	case *ast.UnaryExpr:
+		if t.Op == token.RANGE {
+			// the range operator is only allowed at the top of a for statement
+			p.errorExpected(x.Pos(), "expression")
+			x = &ast.BadExpr{x.Pos(), x.End()}
+		}
+	case *ast.ArrayType:
+		if len, isEllipsis := t.Len.(*ast.Ellipsis); isEllipsis {
+			p.error(len.Pos(), "expected array length, found '...'")
+			x = &ast.BadExpr{x.Pos(), x.End()}
+		}
+	}
+
+	// all other nodes are expressions or types
+	return x
+}
+
+
+func (p *parser) parsePrimaryExpr() ast.Expr {
+	if p.trace {
+		defer un(trace(p, "PrimaryExpr"))
+	}
+
+	x := p.parseOperand()
+L:
+	for {
+		switch p.tok {
+		case token.PERIOD:
+			x = p.parseSelectorOrTypeAssertion(p.checkExpr(x))
+		case token.LBRACK:
+			x = p.parseIndexOrSlice(p.checkExpr(x))
+		case token.LPAREN:
+			x = p.parseCallOrConversion(p.checkExprOrType(x))
+		case token.LBRACE:
+			if isLiteralType(x) && (p.exprLev >= 0 || !isTypeName(x)) {
+				x = p.parseLiteralValue(x)
+			} else {
+				break L
+			}
+		default:
+			break L
+		}
+	}
+
+	return x
+}
+
+
+func (p *parser) parseUnaryExpr() ast.Expr {
+	if p.trace {
+		defer un(trace(p, "UnaryExpr"))
+	}
+
+	switch p.tok {
+	case token.ADD, token.SUB, token.NOT, token.XOR, token.AND, token.RANGE:
+		pos, op := p.pos, p.tok
+		p.next()
+		x := p.parseUnaryExpr()
+		return &ast.UnaryExpr{pos, op, p.checkExpr(x)}
+
+	case token.ARROW:
+		// channel type or receive expression
+		pos := p.pos
+		p.next()
+		if p.tok == token.CHAN {
+			p.next()
+			value := p.parseType()
+			return &ast.ChanType{pos, ast.RECV, value}
+		}
+
+		x := p.parseUnaryExpr()
+		return &ast.UnaryExpr{pos, token.ARROW, p.checkExpr(x)}
+
+	case token.MUL:
+		// pointer type or unary "*" expression
+		pos := p.pos
+		p.next()
+		x := p.parseUnaryExpr()
+		return &ast.StarExpr{pos, p.checkExprOrType(x)}
+	}
+
+	return p.parsePrimaryExpr()
+}
+
+
+func (p *parser) parseBinaryExpr(prec1 int) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "BinaryExpr"))
+	}
+
+	x := p.parseUnaryExpr()
+	for prec := p.tok.Precedence(); prec >= prec1; prec-- {
+		for p.tok.Precedence() == prec {
+			pos, op := p.pos, p.tok
+			p.next()
+			y := p.parseBinaryExpr(prec + 1)
+			x = &ast.BinaryExpr{p.checkExpr(x), pos, op, p.checkExpr(y)}
+		}
+	}
+
+	return x
+}
+
+
+// TODO(gri): parseExpr may return a type or even a raw type ([..]int) -
+//            should reject when a type/raw type is obviously not allowed
+func (p *parser) parseExpr() ast.Expr {
+	if p.trace {
+		defer un(trace(p, "Expression"))
+	}
+
+	return p.parseBinaryExpr(token.LowestPrec + 1)
+}
+
+
+// ----------------------------------------------------------------------------
+// Statements
+
+func (p *parser) parseSimpleStmt(labelOk bool) ast.Stmt {
+	if p.trace {
+		defer un(trace(p, "SimpleStmt"))
+	}
+
+	x := p.parseExprList()
+
+	switch p.tok {
+	case token.COLON:
+		// labeled statement
+		colon := p.pos
+		p.next()
+		if labelOk && len(x) == 1 {
+			if label, isIdent := x[0].(*ast.Ident); isIdent {
+				return &ast.LabeledStmt{label, colon, p.parseStmt()}
+			}
+		}
+		p.error(x[0].Pos(), "illegal label declaration")
+		return &ast.BadStmt{x[0].Pos(), colon + 1}
+
+	case
+		token.DEFINE, token.ASSIGN, token.ADD_ASSIGN,
+		token.SUB_ASSIGN, token.MUL_ASSIGN, token.QUO_ASSIGN,
+		token.REM_ASSIGN, token.AND_ASSIGN, token.OR_ASSIGN,
+		token.XOR_ASSIGN, token.SHL_ASSIGN, token.SHR_ASSIGN, token.AND_NOT_ASSIGN:
+		// assignment statement
+		pos, tok := p.pos, p.tok
+		p.next()
+		y := p.parseExprList()
+		return &ast.AssignStmt{x, pos, tok, y}
+	}
+
+	if len(x) > 1 {
+		p.error(x[0].Pos(), "only one expression allowed")
+		// continue with first expression
+	}
+
+	if p.tok == token.INC || p.tok == token.DEC {
+		// increment or decrement
+		s := &ast.IncDecStmt{x[0], p.pos, p.tok}
+		p.next() // consume "++" or "--"
+		return s
+	}
+
+	// expression
+	return &ast.ExprStmt{x[0]}
+}
+
+
+func (p *parser) parseCallExpr() *ast.CallExpr {
+	x := p.parseExpr()
+	if call, isCall := x.(*ast.CallExpr); isCall {
+		return call
+	}
+	p.errorExpected(x.Pos(), "function/method call")
+	return nil
+}
+
+
+func (p *parser) parseGoStmt() ast.Stmt {
+	if p.trace {
+		defer un(trace(p, "GoStmt"))
+	}
+
+	pos := p.expect(token.GO)
+	call := p.parseCallExpr()
+	p.expectSemi()
+	if call == nil {
+		return &ast.BadStmt{pos, pos + 2} // len("go")
+	}
+
+	return &ast.GoStmt{pos, call}
+}
+
+
+func (p *parser) parseDeferStmt() ast.Stmt {
+	if p.trace {
+		defer un(trace(p, "DeferStmt"))
+	}
+
+	pos := p.expect(token.DEFER)
+	call := p.parseCallExpr()
+	p.expectSemi()
+	if call == nil {
+		return &ast.BadStmt{pos, pos + 5} // len("defer")
+	}
+
+	return &ast.DeferStmt{pos, call}
+}
+
+
+func (p *parser) parseReturnStmt() *ast.ReturnStmt {
+	if p.trace {
+		defer un(trace(p, "ReturnStmt"))
+	}
+
+	pos := p.pos
+	p.expect(token.RETURN)
+	var x []ast.Expr
+	if p.tok != token.SEMICOLON && p.tok != token.RBRACE {
+		x = p.parseExprList()
+	}
+	p.expectSemi()
+
+	return &ast.ReturnStmt{pos, x}
+}
+
+
+func (p *parser) parseBranchStmt(tok token.Token) *ast.BranchStmt {
+	if p.trace {
+		defer un(trace(p, "BranchStmt"))
+	}
+
+	s := &ast.BranchStmt{p.pos, tok, nil}
+	p.expect(tok)
+	if tok != token.FALLTHROUGH && p.tok == token.IDENT {
+		s.Label = p.parseIdent()
+	}
+	p.expectSemi()
+
+	return s
+}
+
+
+func (p *parser) makeExpr(s ast.Stmt) ast.Expr {
+	if s == nil {
+		return nil
+	}
+	if es, isExpr := s.(*ast.ExprStmt); isExpr {
+		return p.checkExpr(es.X)
+	}
+	p.error(s.Pos(), "expected condition, found simple statement")
+	return &ast.BadExpr{s.Pos(), s.End()}
+}
+
+
+func (p *parser) parseControlClause(isForStmt bool) (s1, s2, s3 ast.Stmt) {
+	if p.tok != token.LBRACE {
+		prevLev := p.exprLev
+		p.exprLev = -1
+
+		if p.tok != token.SEMICOLON {
+			s1 = p.parseSimpleStmt(false)
+		}
+		if p.tok == token.SEMICOLON {
+			p.next()
+			if p.tok != token.LBRACE && p.tok != token.SEMICOLON {
+				s2 = p.parseSimpleStmt(false)
+			}
+			if isForStmt {
+				// for statements have a 3rd section
+				p.expectSemi()
+				if p.tok != token.LBRACE {
+					s3 = p.parseSimpleStmt(false)
+				}
+			}
+		} else {
+			s1, s2 = nil, s1
+		}
+
+		p.exprLev = prevLev
+	}
+
+	return s1, s2, s3
+}
+
+
+func (p *parser) parseIfStmt() *ast.IfStmt {
+	if p.trace {
+		defer un(trace(p, "IfStmt"))
+	}
+
+	pos := p.expect(token.IF)
+	s1, s2, _ := p.parseControlClause(false)
+	body := p.parseBlockStmt()
+	var else_ ast.Stmt
+	if p.tok == token.ELSE {
+		p.next()
+		else_ = p.parseStmt()
+	} else {
+		p.expectSemi()
+	}
+
+	return &ast.IfStmt{pos, s1, p.makeExpr(s2), body, else_}
+}
+
+
+func (p *parser) parseCaseClause() *ast.CaseClause {
+	if p.trace {
+		defer un(trace(p, "CaseClause"))
+	}
+
+	// SwitchCase
+	pos := p.pos
+	var x []ast.Expr
+	if p.tok == token.CASE {
+		p.next()
+		x = p.parseExprList()
+	} else {
+		p.expect(token.DEFAULT)
+	}
+
+	colon := p.expect(token.COLON)
+	body := p.parseStmtList()
+
+	return &ast.CaseClause{pos, x, colon, body}
+}
+
+
+func (p *parser) parseTypeList() (list []ast.Expr) {
+	if p.trace {
+		defer un(trace(p, "TypeList"))
+	}
+
+	list = append(list, p.parseType())
+	for p.tok == token.COMMA {
+		p.next()
+		list = append(list, p.parseType())
+	}
+
+	return
+}
+
+
+func (p *parser) parseTypeCaseClause() *ast.TypeCaseClause {
+	if p.trace {
+		defer un(trace(p, "TypeCaseClause"))
+	}
+
+	// TypeSwitchCase
+	pos := p.pos
+	var types []ast.Expr
+	if p.tok == token.CASE {
+		p.next()
+		types = p.parseTypeList()
+	} else {
+		p.expect(token.DEFAULT)
+	}
+
+	colon := p.expect(token.COLON)
+	body := p.parseStmtList()
+
+	return &ast.TypeCaseClause{pos, types, colon, body}
+}
+
+
+func isExprSwitch(s ast.Stmt) bool {
+	if s == nil {
+		return true
+	}
+	if e, ok := s.(*ast.ExprStmt); ok {
+		if a, ok := e.X.(*ast.TypeAssertExpr); ok {
+			return a.Type != nil // regular type assertion
+		}
+		return true
+	}
+	return false
+}
+
+
+func (p *parser) parseSwitchStmt() ast.Stmt {
+	if p.trace {
+		defer un(trace(p, "SwitchStmt"))
+	}
+
+	pos := p.expect(token.SWITCH)
+	s1, s2, _ := p.parseControlClause(false)
+
+	if isExprSwitch(s2) {
+		lbrace := p.expect(token.LBRACE)
+		var list []ast.Stmt
+		for p.tok == token.CASE || p.tok == token.DEFAULT {
+			list = append(list, p.parseCaseClause())
+		}
+		rbrace := p.expect(token.RBRACE)
+		body := &ast.BlockStmt{lbrace, list, rbrace}
+		p.expectSemi()
+		return &ast.SwitchStmt{pos, s1, p.makeExpr(s2), body}
+	}
+
+	// type switch
+	// TODO(gri): do all the checks!
+	lbrace := p.expect(token.LBRACE)
+	var list []ast.Stmt
+	for p.tok == token.CASE || p.tok == token.DEFAULT {
+		list = append(list, p.parseTypeCaseClause())
+	}
+	rbrace := p.expect(token.RBRACE)
+	p.expectSemi()
+	body := &ast.BlockStmt{lbrace, list, rbrace}
+	return &ast.TypeSwitchStmt{pos, s1, s2, body}
+}
+
+
+func (p *parser) parseCommClause() *ast.CommClause {
+	if p.trace {
+		defer un(trace(p, "CommClause"))
+	}
+
+	// CommCase
+	pos := p.pos
+	var tok token.Token
+	var lhs, rhs ast.Expr
+	if p.tok == token.CASE {
+		p.next()
+		if p.tok == token.ARROW {
+			// RecvExpr without assignment
+			rhs = p.parseExpr()
+		} else {
+			// SendExpr or RecvExpr
+			rhs = p.parseExpr()
+			if p.tok == token.ASSIGN || p.tok == token.DEFINE {
+				// RecvExpr with assignment
+				tok = p.tok
+				p.next()
+				lhs = rhs
+				if p.tok == token.ARROW {
+					rhs = p.parseExpr()
+				} else {
+					p.expect(token.ARROW) // use expect() error handling
+				}
+			}
+			// else SendExpr
+		}
+	} else {
+		p.expect(token.DEFAULT)
+	}
+
+	colon := p.expect(token.COLON)
+	body := p.parseStmtList()
+
+	return &ast.CommClause{pos, tok, lhs, rhs, colon, body}
+}
+
+
+func (p *parser) parseSelectStmt() *ast.SelectStmt {
+	if p.trace {
+		defer un(trace(p, "SelectStmt"))
+	}
+
+	pos := p.expect(token.SELECT)
+	lbrace := p.expect(token.LBRACE)
+	var list []ast.Stmt
+	for p.tok == token.CASE || p.tok == token.DEFAULT {
+		list = append(list, p.parseCommClause())
+	}
+	rbrace := p.expect(token.RBRACE)
+	p.expectSemi()
+	body := &ast.BlockStmt{lbrace, list, rbrace}
+
+	return &ast.SelectStmt{pos, body}
+}
+
+
+func (p *parser) parseForStmt() ast.Stmt {
+	if p.trace {
+		defer un(trace(p, "ForStmt"))
+	}
+
+	pos := p.expect(token.FOR)
+	s1, s2, s3 := p.parseControlClause(true)
+	body := p.parseBlockStmt()
+	p.expectSemi()
+
+	if as, isAssign := s2.(*ast.AssignStmt); isAssign {
+		// possibly a for statement with a range clause; check assignment operator
+		if as.Tok != token.ASSIGN && as.Tok != token.DEFINE {
+			p.errorExpected(as.TokPos, "'=' or ':='")
+			return &ast.BadStmt{pos, body.End()}
+		}
+		// check lhs
+		var key, value ast.Expr
+		switch len(as.Lhs) {
+		case 2:
+			key, value = as.Lhs[0], as.Lhs[1]
+		case 1:
+			key = as.Lhs[0]
+		default:
+			p.errorExpected(as.Lhs[0].Pos(), "1 or 2 expressions")
+			return &ast.BadStmt{pos, body.End()}
+		}
+		// check rhs
+		if len(as.Rhs) != 1 {
+			p.errorExpected(as.Rhs[0].Pos(), "1 expressions")
+			return &ast.BadStmt{pos, body.End()}
+		}
+		if rhs, isUnary := as.Rhs[0].(*ast.UnaryExpr); isUnary && rhs.Op == token.RANGE {
+			// rhs is range expression; check lhs
+			return &ast.RangeStmt{pos, key, value, as.TokPos, as.Tok, rhs.X, body}
+		} else {
+			p.errorExpected(s2.Pos(), "range clause")
+			return &ast.BadStmt{pos, body.End()}
+		}
+	} else {
+		// regular for statement
+		return &ast.ForStmt{pos, s1, p.makeExpr(s2), s3, body}
+	}
+
+	panic("unreachable")
+}
+
+
+func (p *parser) parseStmt() (s ast.Stmt) {
+	if p.trace {
+		defer un(trace(p, "Statement"))
+	}
+
+	switch p.tok {
+	case token.CONST, token.TYPE, token.VAR:
+		s = &ast.DeclStmt{p.parseDecl()}
+	case
+		// tokens that may start a top-level expression
+		token.IDENT, token.INT, token.FLOAT, token.CHAR, token.STRING, token.FUNC, token.LPAREN, // operand
+		token.LBRACK, token.STRUCT, // composite type
+		token.MUL, token.AND, token.ARROW, token.ADD, token.SUB, token.XOR: // unary operators
+		s = p.parseSimpleStmt(true)
+		// because of the required look-ahead, labeled statements are
+		// parsed by parseSimpleStmt - don't expect a semicolon after
+		// them
+		if _, isLabeledStmt := s.(*ast.LabeledStmt); !isLabeledStmt {
+			p.expectSemi()
+		}
+	case token.GO:
+		s = p.parseGoStmt()
+	case token.DEFER:
+		s = p.parseDeferStmt()
+	case token.RETURN:
+		s = p.parseReturnStmt()
+	case token.BREAK, token.CONTINUE, token.GOTO, token.FALLTHROUGH:
+		s = p.parseBranchStmt(p.tok)
+	case token.LBRACE:
+		s = p.parseBlockStmt()
+		p.expectSemi()
+	case token.IF:
+		s = p.parseIfStmt()
+	case token.SWITCH:
+		s = p.parseSwitchStmt()
+	case token.SELECT:
+		s = p.parseSelectStmt()
+	case token.FOR:
+		s = p.parseForStmt()
+	case token.SEMICOLON:
+		s = &ast.EmptyStmt{p.pos}
+		p.next()
+	case token.RBRACE:
+		// a semicolon may be omitted before a closing "}"
+		s = &ast.EmptyStmt{p.pos}
+	default:
+		// no statement found
+		pos := p.pos
+		p.errorExpected(pos, "statement")
+		p.next() // make progress
+		s = &ast.BadStmt{pos, p.pos}
+	}
+
+	return
+}
+
+
+// ----------------------------------------------------------------------------
+// Declarations
+
+type parseSpecFunction func(p *parser, doc *ast.CommentGroup) ast.Spec
+
+
+func parseImportSpec(p *parser, doc *ast.CommentGroup) ast.Spec {
+	if p.trace {
+		defer un(trace(p, "ImportSpec"))
+	}
+
+	var ident *ast.Ident
+	if p.tok == token.PERIOD {
+		ident = &ast.Ident{p.pos, ".", nil}
+		p.next()
+	} else if p.tok == token.IDENT {
+		ident = p.parseIdent()
+	}
+
+	var path *ast.BasicLit
+	if p.tok == token.STRING {
+		path = &ast.BasicLit{p.pos, p.tok, p.lit}
+		p.next()
+	} else {
+		p.expect(token.STRING) // use expect() error handling
+	}
+	p.expectSemi()
+
+	return &ast.ImportSpec{doc, ident, path, p.lineComment}
+}
+
+
+func parseConstSpec(p *parser, doc *ast.CommentGroup) ast.Spec {
+	if p.trace {
+		defer un(trace(p, "ConstSpec"))
+	}
+
+	idents := p.parseIdentList()
+	typ := p.tryType()
+	var values []ast.Expr
+	if typ != nil || p.tok == token.ASSIGN {
+		p.expect(token.ASSIGN)
+		values = p.parseExprList()
+	}
+	p.expectSemi()
+
+	return &ast.ValueSpec{doc, idents, typ, values, p.lineComment}
+}
+
+
+func parseTypeSpec(p *parser, doc *ast.CommentGroup) ast.Spec {
+	if p.trace {
+		defer un(trace(p, "TypeSpec"))
+	}
+
+	ident := p.parseIdent()
+	typ := p.parseType()
+	p.expectSemi()
+
+	return &ast.TypeSpec{doc, ident, typ, p.lineComment}
+}
+
+
+func parseVarSpec(p *parser, doc *ast.CommentGroup) ast.Spec {
+	if p.trace {
+		defer un(trace(p, "VarSpec"))
+	}
+
+	idents := p.parseIdentList()
+	typ := p.tryType()
+	var values []ast.Expr
+	if typ == nil || p.tok == token.ASSIGN {
+		p.expect(token.ASSIGN)
+		values = p.parseExprList()
+	}
+	p.expectSemi()
+
+	return &ast.ValueSpec{doc, idents, typ, values, p.lineComment}
+}
+
+
+func (p *parser) parseGenDecl(keyword token.Token, f parseSpecFunction) *ast.GenDecl {
+	if p.trace {
+		defer un(trace(p, "GenDecl("+keyword.String()+")"))
+	}
+
+	doc := p.leadComment
+	pos := p.expect(keyword)
+	var lparen, rparen token.Pos
+	var list []ast.Spec
+	if p.tok == token.LPAREN {
+		lparen = p.pos
+		p.next()
+		for p.tok != token.RPAREN && p.tok != token.EOF {
+			list = append(list, f(p, p.leadComment))
+		}
+		rparen = p.expect(token.RPAREN)
+		p.expectSemi()
+	} else {
+		list = append(list, f(p, nil))
+	}
+
+	return &ast.GenDecl{doc, pos, keyword, lparen, list, rparen}
+}
+
+
+func (p *parser) parseReceiver() *ast.FieldList {
+	if p.trace {
+		defer un(trace(p, "Receiver"))
+	}
+
+	pos := p.pos
+	par := p.parseParameters(false)
+
+	// must have exactly one receiver
+	if par.NumFields() != 1 {
+		p.errorExpected(pos, "exactly one receiver")
+		// TODO determine a better range for BadExpr below
+		par.List = []*ast.Field{&ast.Field{Type: &ast.BadExpr{pos, pos}}}
+		return par
+	}
+
+	// recv type must be of the form ["*"] identifier
+	recv := par.List[0]
+	base := deref(recv.Type)
+	if _, isIdent := base.(*ast.Ident); !isIdent {
+		p.errorExpected(base.Pos(), "(unqualified) identifier")
+		par.List = []*ast.Field{&ast.Field{Type: &ast.BadExpr{recv.Pos(), recv.End()}}}
+	}
+
+	return par
+}
+
+
+func (p *parser) parseFuncDecl() *ast.FuncDecl {
+	if p.trace {
+		defer un(trace(p, "FunctionDecl"))
+	}
+
+	doc := p.leadComment
+	pos := p.expect(token.FUNC)
+
+	var recv *ast.FieldList
+	if p.tok == token.LPAREN {
+		recv = p.parseReceiver()
+	}
+
+	ident := p.parseIdent()
+	params, results := p.parseSignature()
+
+	var body *ast.BlockStmt
+	if p.tok == token.LBRACE {
+		body = p.parseBody()
+	}
+	p.expectSemi()
+
+	return &ast.FuncDecl{doc, recv, ident, &ast.FuncType{pos, params, results}, body}
+}
+
+
+func (p *parser) parseDecl() ast.Decl {
+	if p.trace {
+		defer un(trace(p, "Declaration"))
+	}
+
+	var f parseSpecFunction
+	switch p.tok {
+	case token.CONST:
+		f = parseConstSpec
+
+	case token.TYPE:
+		f = parseTypeSpec
+
+	case token.VAR:
+		f = parseVarSpec
+
+	case token.FUNC:
+		return p.parseFuncDecl()
+
+	default:
+		pos := p.pos
+		p.errorExpected(pos, "declaration")
+		p.next() // make progress
+		decl := &ast.BadDecl{pos, p.pos}
+		return decl
+	}
+
+	return p.parseGenDecl(p.tok, f)
+}
+
+
+func (p *parser) parseDeclList() (list []ast.Decl) {
+	if p.trace {
+		defer un(trace(p, "DeclList"))
+	}
+
+	for p.tok != token.EOF {
+		list = append(list, p.parseDecl())
+	}
+
+	return
+}
+
+
+// ----------------------------------------------------------------------------
+// Source files
+
+func (p *parser) parseFile() *ast.File {
+	if p.trace {
+		defer un(trace(p, "File"))
+	}
+
+	// package clause
+	doc := p.leadComment
+	pos := p.expect(token.PACKAGE)
+	ident := p.parseIdent()
+	p.expectSemi()
+
+	var decls []ast.Decl
+
+	// Don't bother parsing the rest if we had errors already.
+	// Likely not a Go source file at all.
+
+	if p.ErrorCount() == 0 && p.mode&PackageClauseOnly == 0 {
+		// import decls
+		for p.tok == token.IMPORT {
+			decls = append(decls, p.parseGenDecl(token.IMPORT, parseImportSpec))
+		}
+
+		if p.mode&ImportsOnly == 0 {
+			// rest of package body
+			for p.tok != token.EOF {
+				decls = append(decls, p.parseDecl())
+			}
+		}
+	}
+
+	return &ast.File{doc, pos, ident, decls, p.comments}
+}
diff --git a/libgo/go/go/parser/parser_test.go b/libgo/go/go/parser/parser_test.go
new file mode 100644
index 000000000..56bd80ef1
--- /dev/null
+++ b/libgo/go/go/parser/parser_test.go
@@ -0,0 +1,112 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package parser
+
+import (
+	"go/token"
+	"os"
+	"testing"
+)
+
+
+var fset = token.NewFileSet()
+
+var illegalInputs = []interface{}{
+	nil,
+	3.14,
+	[]byte(nil),
+	"foo!",
+}
+
+
+func TestParseIllegalInputs(t *testing.T) {
+	for _, src := range illegalInputs {
+		_, err := ParseFile(fset, "", src, 0)
+		if err == nil {
+			t.Errorf("ParseFile(%v) should have failed", src)
+		}
+	}
+}
+
+
+var validPrograms = []interface{}{
+	"package main\n",
+	`package main;`,
+	`package main; import "fmt"; func main() { fmt.Println("Hello, World!") };`,
+	`package main; func main() { if f(T{}) {} };`,
+	`package main; func main() { _ = (<-chan int)(x) };`,
+	`package main; func main() { _ = (<-chan <-chan int)(x) };`,
+	`package main; func f(func() func() func());`,
+	`package main; func f(...T);`,
+	`package main; func f(float, ...int);`,
+	`package main; func f(x int, a ...int) { f(0, a...); f(1, a...,) };`,
+	`package main; type T []int; var a []bool; func f() { if a[T{42}[0]] {} };`,
+	`package main; type T []int; func g(int) bool { return true }; func f() { if g(T{42}[0]) {} };`,
+	`package main; type T []int; func f() { for _ = range []int{T{42}[0]} {} };`,
+	`package main; var a = T{{1, 2}, {3, 4}}`,
+}
+
+
+func TestParseValidPrograms(t *testing.T) {
+	for _, src := range validPrograms {
+		_, err := ParseFile(fset, "", src, 0)
+		if err != nil {
+			t.Errorf("ParseFile(%q): %v", src, err)
+		}
+	}
+}
+
+
+var validFiles = []string{
+	"parser.go",
+	"parser_test.go",
+}
+
+
+func TestParse3(t *testing.T) {
+	for _, filename := range validFiles {
+		_, err := ParseFile(fset, filename, nil, 0)
+		if err != nil {
+			t.Errorf("ParseFile(%s): %v", filename, err)
+		}
+	}
+}
+
+
+func nameFilter(filename string) bool {
+	switch filename {
+	case "parser.go":
+	case "interface.go":
+	case "parser_test.go":
+	default:
+		return false
+	}
+	return true
+}
+
+
+func dirFilter(f *os.FileInfo) bool { return nameFilter(f.Name) }
+
+
+func TestParse4(t *testing.T) {
+	path := "."
+	pkgs, err := ParseDir(fset, path, dirFilter, 0)
+	if err != nil {
+		t.Fatalf("ParseDir(%s): %v", path, err)
+	}
+	if len(pkgs) != 1 {
+		t.Errorf("incorrect number of packages: %d", len(pkgs))
+	}
+	pkg := pkgs["parser"]
+	if pkg == nil {
+		t.Errorf(`package "parser" not found`)
+		return
+	}
+	for filename := range pkg.Files {
+		if !nameFilter(filename) {
+			t.Errorf("unexpected package file: %s", filename)
+		}
+	}
+}
diff --git a/libgo/go/go/printer/nodes.go b/libgo/go/go/printer/nodes.go
new file mode 100644
index 000000000..8207996dc
--- /dev/null
+++ b/libgo/go/go/printer/nodes.go
@@ -0,0 +1,1493 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This file implements printing of AST nodes; specifically
+// expressions, statements, declarations, and files. It uses
+// the print functionality implemented in printer.go.
+
+package printer
+
+import (
+	"bytes"
+	"go/ast"
+	"go/token"
+)
+
+
+// Other formatting issues:
+// - better comment formatting for /*-style comments at the end of a line (e.g. a declaration)
+//   when the comment spans multiple lines; if such a comment is just two lines, formatting is
+//   not idempotent
+// - formatting of expression lists
+// - should use blank instead of tab to separate one-line function bodies from
+//   the function header unless there is a group of consecutive one-liners
+
+
+// ----------------------------------------------------------------------------
+// Common AST nodes.
+
+// Print as many newlines as necessary (but at least min newlines) to get to
+// the current line. ws is printed before the first line break. If newSection
+// is set, the first line break is printed as formfeed. Returns true if any
+// line break was printed; returns false otherwise.
+//
+// TODO(gri): linebreak may add too many lines if the next statement at "line"
+//            is preceeded by comments because the computation of n assumes
+//            the current position before the comment and the target position
+//            after the comment. Thus, after interspersing such comments, the
+//            space taken up by them is not considered to reduce the number of
+//            linebreaks. At the moment there is no easy way to know about
+//            future (not yet interspersed) comments in this function.
+//
+func (p *printer) linebreak(line, min int, ws whiteSpace, newSection bool) (printedBreak bool) {
+	n := p.nlines(line-p.pos.Line, min)
+	if n > 0 {
+		p.print(ws)
+		if newSection {
+			p.print(formfeed)
+			n--
+		}
+		for ; n > 0; n-- {
+			p.print(newline)
+		}
+		printedBreak = true
+	}
+	return
+}
+
+
+// setComment sets g as the next comment if g != nil and if node comments
+// are enabled - this mode is used when printing source code fragments such
+// as exports only. It assumes that there are no other pending comments to
+// intersperse.
+func (p *printer) setComment(g *ast.CommentGroup) {
+	if g == nil || !p.useNodeComments {
+		return
+	}
+	if p.comments == nil {
+		// initialize p.comments lazily
+		p.comments = make([]*ast.CommentGroup, 1)
+	} else if p.cindex < len(p.comments) {
+		// for some reason there are pending comments; this
+		// should never happen - handle gracefully and flush
+		// all comments up to g, ignore anything after that
+		p.flush(p.fset.Position(g.List[0].Pos()), token.ILLEGAL)
+	}
+	p.comments[0] = g
+	p.cindex = 0
+}
+
+
+type exprListMode uint
+
+const (
+	blankStart exprListMode = 1 << iota // print a blank before a non-empty list
+	blankEnd                            // print a blank after a non-empty list
+	commaSep                            // elements are separated by commas
+	commaTerm                           // list is optionally terminated by a comma
+	noIndent                            // no extra indentation in multi-line lists
+	periodSep                           // elements are separated by periods
+)
+
+
+// Sets multiLine to true if the identifier list spans multiple lines.
+// If indent is set, a multi-line identifier list is indented after the
+// first linebreak encountered.
+func (p *printer) identList(list []*ast.Ident, indent bool, multiLine *bool) {
+	// convert into an expression list so we can re-use exprList formatting
+	xlist := make([]ast.Expr, len(list))
+	for i, x := range list {
+		xlist[i] = x
+	}
+	mode := commaSep
+	if !indent {
+		mode |= noIndent
+	}
+	p.exprList(token.NoPos, xlist, 1, mode, multiLine, token.NoPos)
+}
+
+
+// Compute the key size of a key:value expression.
+// Returns 0 if the expression doesn't fit onto a single line.
+func (p *printer) keySize(pair *ast.KeyValueExpr) int {
+	if p.nodeSize(pair, infinity) <= infinity {
+		// entire expression fits on one line - return key size
+		return p.nodeSize(pair.Key, infinity)
+	}
+	return 0
+}
+
+
+// Print a list of expressions. If the list spans multiple
+// source lines, the original line breaks are respected between
+// expressions. Sets multiLine to true if the list spans multiple
+// lines.
+//
+// TODO(gri) Consider rewriting this to be independent of []ast.Expr
+//           so that we can use the algorithm for any kind of list
+//           (e.g., pass list via a channel over which to range).
+func (p *printer) exprList(prev0 token.Pos, list []ast.Expr, depth int, mode exprListMode, multiLine *bool, next0 token.Pos) {
+	if len(list) == 0 {
+		return
+	}
+
+	if mode&blankStart != 0 {
+		p.print(blank)
+	}
+
+	prev := p.fset.Position(prev0)
+	next := p.fset.Position(next0)
+	line := p.fset.Position(list[0].Pos()).Line
+	endLine := p.fset.Position(list[len(list)-1].End()).Line
+
+	if prev.IsValid() && prev.Line == line && line == endLine {
+		// all list entries on a single line
+		for i, x := range list {
+			if i > 0 {
+				if mode&commaSep != 0 {
+					p.print(token.COMMA)
+				}
+				p.print(blank)
+			}
+			p.expr0(x, depth, multiLine)
+		}
+		if mode&blankEnd != 0 {
+			p.print(blank)
+		}
+		return
+	}
+
+	// list entries span multiple lines;
+	// use source code positions to guide line breaks
+
+	// don't add extra indentation if noIndent is set;
+	// i.e., pretend that the first line is already indented
+	ws := ignore
+	if mode&noIndent == 0 {
+		ws = indent
+	}
+
+	// the first linebreak is always a formfeed since this section must not
+	// depend on any previous formatting
+	prevBreak := -1 // index of last expression that was followed by a linebreak
+	linebreakMin := 1
+	if mode&periodSep != 0 {
+		// Make fragments like
+		//
+		// a.Bar(1,
+		//   2).Foo
+		//
+		// format correctly (a linebreak shouldn't be added before Foo) when
+		// doing period-separated expr lists by setting minimum linebreak to 0
+		// lines for them.
+		linebreakMin = 0
+	}
+	if prev.IsValid() && prev.Line < line && p.linebreak(line, linebreakMin, ws, true) {
+		ws = ignore
+		*multiLine = true
+		prevBreak = 0
+	}
+
+	// initialize expression/key size: a zero value indicates expr/key doesn't fit on a single line
+	size := 0
+
+	// print all list elements
+	for i, x := range list {
+		prevLine := line
+		line = p.fset.Position(x.Pos()).Line
+
+		// determine if the next linebreak, if any, needs to use formfeed:
+		// in general, use the entire node size to make the decision; for
+		// key:value expressions, use the key size
+		// TODO(gri) for a better result, should probably incorporate both
+		//           the key and the node size into the decision process
+		useFF := true
+
+		// determine size
+		prevSize := size
+		const infinity = 1e6 // larger than any source line
+		size = p.nodeSize(x, infinity)
+		pair, isPair := x.(*ast.KeyValueExpr)
+		if size <= infinity {
+			// x fits on a single line
+			if isPair {
+				size = p.nodeSize(pair.Key, infinity) // size <= infinity
+			}
+		} else {
+			size = 0
+		}
+
+		// if the previous line and the current line had single-
+		// line-expressions and the key sizes are small or the
+		// the ratio between the key sizes does not exceed a
+		// threshold, align columns and do not use formfeed
+		if prevSize > 0 && size > 0 {
+			const smallSize = 20
+			if prevSize <= smallSize && size <= smallSize {
+				useFF = false
+			} else {
+				const r = 4 // threshold
+				ratio := float64(size) / float64(prevSize)
+				useFF = ratio <= 1/r || r <= ratio
+			}
+		}
+
+		if i > 0 {
+			if mode&commaSep != 0 {
+				p.print(token.COMMA)
+			}
+			if mode&periodSep != 0 {
+				p.print(token.PERIOD)
+			}
+			if prevLine < line && prevLine > 0 && line > 0 {
+				// lines are broken using newlines so comments remain aligned
+				// unless forceFF is set or there are multiple expressions on
+				// the same line in which case formfeed is used
+				// broken with a formfeed
+				if p.linebreak(line, linebreakMin, ws, useFF || prevBreak+1 < i) {
+					ws = ignore
+					*multiLine = true
+					prevBreak = i
+				}
+			} else if mode&periodSep == 0 {
+				p.print(blank)
+			}
+			// period-separated list elements don't need a blank
+		}
+
+		if isPair && size > 0 && len(list) > 1 {
+			// we have a key:value expression that fits onto one line and
+			// is in a list with more then one entry: use a column for the
+			// key such that consecutive entries can align if possible
+			p.expr(pair.Key, multiLine)
+			p.print(pair.Colon, token.COLON, vtab)
+			p.expr(pair.Value, multiLine)
+		} else {
+			p.expr0(x, depth, multiLine)
+		}
+	}
+
+	if mode&commaTerm != 0 && next.IsValid() && p.pos.Line < next.Line {
+		// print a terminating comma if the next token is on a new line
+		p.print(token.COMMA)
+		if ws == ignore && mode&noIndent == 0 {
+			// unindent if we indented
+			p.print(unindent)
+		}
+		p.print(formfeed) // terminating comma needs a line break to look good
+		return
+	}
+
+	if mode&blankEnd != 0 {
+		p.print(blank)
+	}
+
+	if ws == ignore && mode&noIndent == 0 {
+		// unindent if we indented
+		p.print(unindent)
+	}
+}
+
+
+// Sets multiLine to true if the the parameter list spans multiple lines.
+func (p *printer) parameters(fields *ast.FieldList, multiLine *bool) {
+	p.print(fields.Opening, token.LPAREN)
+	if len(fields.List) > 0 {
+		var prevLine, line int
+		for i, par := range fields.List {
+			if i > 0 {
+				p.print(token.COMMA)
+				if len(par.Names) > 0 {
+					line = p.fset.Position(par.Names[0].Pos()).Line
+				} else {
+					line = p.fset.Position(par.Type.Pos()).Line
+				}
+				if 0 < prevLine && prevLine < line && p.linebreak(line, 0, ignore, true) {
+					*multiLine = true
+				} else {
+					p.print(blank)
+				}
+			}
+			if len(par.Names) > 0 {
+				p.identList(par.Names, false, multiLine)
+				p.print(blank)
+			}
+			p.expr(par.Type, multiLine)
+			prevLine = p.fset.Position(par.Type.Pos()).Line
+		}
+	}
+	p.print(fields.Closing, token.RPAREN)
+}
+
+
+// Sets multiLine to true if the signature spans multiple lines.
+func (p *printer) signature(params, result *ast.FieldList, multiLine *bool) {
+	p.parameters(params, multiLine)
+	n := result.NumFields()
+	if n > 0 {
+		p.print(blank)
+		if n == 1 && result.List[0].Names == nil {
+			// single anonymous result; no ()'s
+			p.expr(result.List[0].Type, multiLine)
+			return
+		}
+		p.parameters(result, multiLine)
+	}
+}
+
+
+func identListSize(list []*ast.Ident, maxSize int) (size int) {
+	for i, x := range list {
+		if i > 0 {
+			size += 2 // ", "
+		}
+		size += len(x.Name)
+		if size >= maxSize {
+			break
+		}
+	}
+	return
+}
+
+
+func (p *printer) isOneLineFieldList(list []*ast.Field) bool {
+	if len(list) != 1 {
+		return false // allow only one field
+	}
+	f := list[0]
+	if f.Tag != nil || f.Comment != nil {
+		return false // don't allow tags or comments
+	}
+	// only name(s) and type
+	const maxSize = 30 // adjust as appropriate, this is an approximate value
+	namesSize := identListSize(f.Names, maxSize)
+	if namesSize > 0 {
+		namesSize = 1 // blank between names and types
+	}
+	typeSize := p.nodeSize(f.Type, maxSize)
+	return namesSize+typeSize <= maxSize
+}
+
+
+func (p *printer) setLineComment(text string) {
+	p.setComment(&ast.CommentGroup{[]*ast.Comment{&ast.Comment{token.NoPos, []byte(text)}}})
+}
+
+
+func (p *printer) fieldList(fields *ast.FieldList, isIncomplete bool, ctxt exprContext) {
+	p.nesting++
+	defer func() {
+		p.nesting--
+	}()
+
+	lbrace := fields.Opening
+	list := fields.List
+	rbrace := fields.Closing
+
+	if !isIncomplete && !p.commentBefore(p.fset.Position(rbrace)) {
+		// possibly a one-line struct/interface
+		if len(list) == 0 {
+			// no blank between keyword and {} in this case
+			p.print(lbrace, token.LBRACE, rbrace, token.RBRACE)
+			return
+		} else if ctxt&(compositeLit|structType) == compositeLit|structType &&
+			p.isOneLineFieldList(list) { // for now ignore interfaces
+			// small enough - print on one line
+			// (don't use identList and ignore source line breaks)
+			p.print(lbrace, token.LBRACE, blank)
+			f := list[0]
+			for i, x := range f.Names {
+				if i > 0 {
+					p.print(token.COMMA, blank)
+				}
+				p.expr(x, ignoreMultiLine)
+			}
+			if len(f.Names) > 0 {
+				p.print(blank)
+			}
+			p.expr(f.Type, ignoreMultiLine)
+			p.print(blank, rbrace, token.RBRACE)
+			return
+		}
+	}
+
+	// at least one entry or incomplete
+	p.print(blank, lbrace, token.LBRACE, indent, formfeed)
+	if ctxt&structType != 0 {
+
+		sep := vtab
+		if len(list) == 1 {
+			sep = blank
+		}
+		var ml bool
+		for i, f := range list {
+			if i > 0 {
+				p.linebreak(p.fset.Position(f.Pos()).Line, 1, ignore, ml)
+			}
+			ml = false
+			extraTabs := 0
+			p.setComment(f.Doc)
+			if len(f.Names) > 0 {
+				// named fields
+				p.identList(f.Names, false, &ml)
+				p.print(sep)
+				p.expr(f.Type, &ml)
+				extraTabs = 1
+			} else {
+				// anonymous field
+				p.expr(f.Type, &ml)
+				extraTabs = 2
+			}
+			if f.Tag != nil {
+				if len(f.Names) > 0 && sep == vtab {
+					p.print(sep)
+				}
+				p.print(sep)
+				p.expr(f.Tag, &ml)
+				extraTabs = 0
+			}
+			if f.Comment != nil {
+				for ; extraTabs > 0; extraTabs-- {
+					p.print(sep)
+				}
+				p.setComment(f.Comment)
+			}
+		}
+		if isIncomplete {
+			if len(list) > 0 {
+				p.print(formfeed)
+			}
+			p.flush(p.fset.Position(rbrace), token.RBRACE) // make sure we don't loose the last line comment
+			p.setLineComment("// contains unexported fields")
+		}
+
+	} else { // interface
+
+		var ml bool
+		for i, f := range list {
+			if i > 0 {
+				p.linebreak(p.fset.Position(f.Pos()).Line, 1, ignore, ml)
+			}
+			ml = false
+			p.setComment(f.Doc)
+			if ftyp, isFtyp := f.Type.(*ast.FuncType); isFtyp {
+				// method
+				p.expr(f.Names[0], &ml)
+				p.signature(ftyp.Params, ftyp.Results, &ml)
+			} else {
+				// embedded interface
+				p.expr(f.Type, &ml)
+			}
+			p.setComment(f.Comment)
+		}
+		if isIncomplete {
+			if len(list) > 0 {
+				p.print(formfeed)
+			}
+			p.flush(p.fset.Position(rbrace), token.RBRACE) // make sure we don't loose the last line comment
+			p.setLineComment("// contains unexported methods")
+		}
+
+	}
+	p.print(unindent, formfeed, rbrace, token.RBRACE)
+}
+
+
+// ----------------------------------------------------------------------------
+// Expressions
+
+// exprContext describes the syntactic environment in which an expression node is printed.
+type exprContext uint
+
+const (
+	compositeLit exprContext = 1 << iota
+	structType
+)
+
+
+func walkBinary(e *ast.BinaryExpr) (has5, has6 bool, maxProblem int) {
+	switch e.Op.Precedence() {
+	case 5:
+		has5 = true
+	case 6:
+		has6 = true
+	}
+
+	switch l := e.X.(type) {
+	case *ast.BinaryExpr:
+		if l.Op.Precedence() < e.Op.Precedence() {
+			// parens will be inserted.
+			// pretend this is an *ast.ParenExpr and do nothing.
+			break
+		}
+		h5, h6, mp := walkBinary(l)
+		has5 = has5 || h5
+		has6 = has6 || h6
+		if maxProblem < mp {
+			maxProblem = mp
+		}
+	}
+
+	switch r := e.Y.(type) {
+	case *ast.BinaryExpr:
+		if r.Op.Precedence() <= e.Op.Precedence() {
+			// parens will be inserted.
+			// pretend this is an *ast.ParenExpr and do nothing.
+			break
+		}
+		h5, h6, mp := walkBinary(r)
+		has5 = has5 || h5
+		has6 = has6 || h6
+		if maxProblem < mp {
+			maxProblem = mp
+		}
+
+	case *ast.StarExpr:
+		if e.Op.String() == "/" {
+			maxProblem = 6
+		}
+
+	case *ast.UnaryExpr:
+		switch e.Op.String() + r.Op.String() {
+		case "/*", "&&", "&^":
+			maxProblem = 6
+		case "++", "--":
+			if maxProblem < 5 {
+				maxProblem = 5
+			}
+		}
+	}
+	return
+}
+
+
+func cutoff(e *ast.BinaryExpr, depth int) int {
+	has5, has6, maxProblem := walkBinary(e)
+	if maxProblem > 0 {
+		return maxProblem + 1
+	}
+	if has5 && has6 {
+		if depth == 1 {
+			return 6
+		}
+		return 5
+	}
+	if depth == 1 {
+		return 7
+	}
+	return 5
+}
+
+
+func diffPrec(expr ast.Expr, prec int) int {
+	x, ok := expr.(*ast.BinaryExpr)
+	if !ok || prec != x.Op.Precedence() {
+		return 1
+	}
+	return 0
+}
+
+
+func reduceDepth(depth int) int {
+	depth--
+	if depth < 1 {
+		depth = 1
+	}
+	return depth
+}
+
+
+// Format the binary expression: decide the cutoff and then format.
+// Let's call depth == 1 Normal mode, and depth > 1 Compact mode.
+// (Algorithm suggestion by Russ Cox.)
+//
+// The precedences are:
+//	6             *  /  %  <<  >>  &  &^
+//	5             +  -  |  ^
+//	4             ==  !=  <  <=  >  >=
+//	3             <-
+//	2             &&
+//	1             ||
+//
+// The only decision is whether there will be spaces around levels 5 and 6.
+// There are never spaces at level 7 (unary), and always spaces at levels 4 and below.
+//
+// To choose the cutoff, look at the whole expression but excluding primary
+// expressions (function calls, parenthesized exprs), and apply these rules:
+//
+//	1) If there is a binary operator with a right side unary operand
+//	   that would clash without a space, the cutoff must be (in order):
+//
+//		/*	7
+//		&&	7
+//		&^	7
+//		++	6
+//		--	6
+//
+//         (Comparison operators always have spaces around them.)
+//
+//	2) If there is a mix of level 6 and level 5 operators, then the cutoff
+//	   is 6 (use spaces to distinguish precedence) in Normal mode
+//	   and 5 (never use spaces) in Compact mode.
+//
+//	3) If there are no level 5 operators or no level 6 operators, then the
+//	   cutoff is 7 (always use spaces) in Normal mode
+//	   and 5 (never use spaces) in Compact mode.
+//
+// Sets multiLine to true if the binary expression spans multiple lines.
+func (p *printer) binaryExpr(x *ast.BinaryExpr, prec1, cutoff, depth int, multiLine *bool) {
+	prec := x.Op.Precedence()
+	if prec < prec1 {
+		// parenthesis needed
+		// Note: The parser inserts an ast.ParenExpr node; thus this case
+		//       can only occur if the AST is created in a different way.
+		p.print(token.LPAREN)
+		p.expr0(x, reduceDepth(depth), multiLine) // parentheses undo one level of depth
+		p.print(token.RPAREN)
+		return
+	}
+
+	printBlank := prec < cutoff
+
+	ws := indent
+	p.expr1(x.X, prec, depth+diffPrec(x.X, prec), 0, multiLine)
+	if printBlank {
+		p.print(blank)
+	}
+	xline := p.pos.Line // before the operator (it may be on the next line!)
+	yline := p.fset.Position(x.Y.Pos()).Line
+	p.print(x.OpPos, x.Op)
+	if xline != yline && xline > 0 && yline > 0 {
+		// at least one line break, but respect an extra empty line
+		// in the source
+		if p.linebreak(yline, 1, ws, true) {
+			ws = ignore
+			*multiLine = true
+			printBlank = false // no blank after line break
+		}
+	}
+	if printBlank {
+		p.print(blank)
+	}
+	p.expr1(x.Y, prec+1, depth+1, 0, multiLine)
+	if ws == ignore {
+		p.print(unindent)
+	}
+}
+
+
+func isBinary(expr ast.Expr) bool {
+	_, ok := expr.(*ast.BinaryExpr)
+	return ok
+}
+
+
+// If the expression contains one or more selector expressions, splits it into
+// two expressions at the rightmost period. Writes entire expr to suffix when
+// selector isn't found. Rewrites AST nodes for calls, index expressions and
+// type assertions, all of which may be found in selector chains, to make them
+// parts of the chain.
+func splitSelector(expr ast.Expr) (body, suffix ast.Expr) {
+	switch x := expr.(type) {
+	case *ast.SelectorExpr:
+		body, suffix = x.X, x.Sel
+		return
+	case *ast.CallExpr:
+		body, suffix = splitSelector(x.Fun)
+		if body != nil {
+			suffix = &ast.CallExpr{suffix, x.Lparen, x.Args, x.Ellipsis, x.Rparen}
+			return
+		}
+	case *ast.IndexExpr:
+		body, suffix = splitSelector(x.X)
+		if body != nil {
+			suffix = &ast.IndexExpr{suffix, x.Lbrack, x.Index, x.Rbrack}
+			return
+		}
+	case *ast.SliceExpr:
+		body, suffix = splitSelector(x.X)
+		if body != nil {
+			suffix = &ast.SliceExpr{suffix, x.Lbrack, x.Low, x.High, x.Rbrack}
+			return
+		}
+	case *ast.TypeAssertExpr:
+		body, suffix = splitSelector(x.X)
+		if body != nil {
+			suffix = &ast.TypeAssertExpr{suffix, x.Type}
+			return
+		}
+	}
+	suffix = expr
+	return
+}
+
+
+// Convert an expression into an expression list split at the periods of
+// selector expressions.
+func selectorExprList(expr ast.Expr) (list []ast.Expr) {
+	// split expression
+	for expr != nil {
+		var suffix ast.Expr
+		expr, suffix = splitSelector(expr)
+		list = append(list, suffix)
+	}
+
+	// reverse list
+	for i, j := 0, len(list)-1; i < j; i, j = i+1, j-1 {
+		list[i], list[j] = list[j], list[i]
+	}
+
+	return
+}
+
+
+// Sets multiLine to true if the expression spans multiple lines.
+func (p *printer) expr1(expr ast.Expr, prec1, depth int, ctxt exprContext, multiLine *bool) {
+	p.print(expr.Pos())
+
+	switch x := expr.(type) {
+	case *ast.BadExpr:
+		p.print("BadExpr")
+
+	case *ast.Ident:
+		p.print(x)
+
+	case *ast.BinaryExpr:
+		if depth < 1 {
+			p.internalError("depth < 1:", depth)
+			depth = 1
+		}
+		p.binaryExpr(x, prec1, cutoff(x, depth), depth, multiLine)
+
+	case *ast.KeyValueExpr:
+		p.expr(x.Key, multiLine)
+		p.print(x.Colon, token.COLON, blank)
+		p.expr(x.Value, multiLine)
+
+	case *ast.StarExpr:
+		const prec = token.UnaryPrec
+		if prec < prec1 {
+			// parenthesis needed
+			p.print(token.LPAREN)
+			p.print(token.MUL)
+			p.expr(x.X, multiLine)
+			p.print(token.RPAREN)
+		} else {
+			// no parenthesis needed
+			p.print(token.MUL)
+			p.expr(x.X, multiLine)
+		}
+
+	case *ast.UnaryExpr:
+		const prec = token.UnaryPrec
+		if prec < prec1 {
+			// parenthesis needed
+			p.print(token.LPAREN)
+			p.expr(x, multiLine)
+			p.print(token.RPAREN)
+		} else {
+			// no parenthesis needed
+			p.print(x.Op)
+			if x.Op == token.RANGE {
+				// TODO(gri) Remove this code if it cannot be reached.
+				p.print(blank)
+			}
+			p.expr1(x.X, prec, depth, 0, multiLine)
+		}
+
+	case *ast.BasicLit:
+		p.print(x)
+
+	case *ast.FuncLit:
+		p.expr(x.Type, multiLine)
+		p.funcBody(x.Body, p.distance(x.Type.Pos(), p.pos), true, multiLine)
+
+	case *ast.ParenExpr:
+		if _, hasParens := x.X.(*ast.ParenExpr); hasParens {
+			// don't print parentheses around an already parenthesized expression
+			// TODO(gri) consider making this more general and incorporate precedence levels
+			p.expr0(x.X, reduceDepth(depth), multiLine) // parentheses undo one level of depth
+		} else {
+			p.print(token.LPAREN)
+			p.expr0(x.X, reduceDepth(depth), multiLine) // parentheses undo one level of depth
+			p.print(x.Rparen, token.RPAREN)
+		}
+
+	case *ast.SelectorExpr:
+		parts := selectorExprList(expr)
+		p.exprList(token.NoPos, parts, depth, periodSep, multiLine, token.NoPos)
+
+	case *ast.TypeAssertExpr:
+		p.expr1(x.X, token.HighestPrec, depth, 0, multiLine)
+		p.print(token.PERIOD, token.LPAREN)
+		if x.Type != nil {
+			p.expr(x.Type, multiLine)
+		} else {
+			p.print(token.TYPE)
+		}
+		p.print(token.RPAREN)
+
+	case *ast.IndexExpr:
+		// TODO(gri): should treat[] like parentheses and undo one level of depth
+		p.expr1(x.X, token.HighestPrec, 1, 0, multiLine)
+		p.print(x.Lbrack, token.LBRACK)
+		p.expr0(x.Index, depth+1, multiLine)
+		p.print(x.Rbrack, token.RBRACK)
+
+	case *ast.SliceExpr:
+		// TODO(gri): should treat[] like parentheses and undo one level of depth
+		p.expr1(x.X, token.HighestPrec, 1, 0, multiLine)
+		p.print(x.Lbrack, token.LBRACK)
+		if x.Low != nil {
+			p.expr0(x.Low, depth+1, multiLine)
+		}
+		// blanks around ":" if both sides exist and either side is a binary expression
+		if depth <= 1 && x.Low != nil && x.High != nil && (isBinary(x.Low) || isBinary(x.High)) {
+			p.print(blank, token.COLON, blank)
+		} else {
+			p.print(token.COLON)
+		}
+		if x.High != nil {
+			p.expr0(x.High, depth+1, multiLine)
+		}
+		p.print(x.Rbrack, token.RBRACK)
+
+	case *ast.CallExpr:
+		if len(x.Args) > 1 {
+			depth++
+		}
+		p.expr1(x.Fun, token.HighestPrec, depth, 0, multiLine)
+		p.print(x.Lparen, token.LPAREN)
+		p.exprList(x.Lparen, x.Args, depth, commaSep|commaTerm, multiLine, x.Rparen)
+		if x.Ellipsis.IsValid() {
+			p.print(x.Ellipsis, token.ELLIPSIS)
+		}
+		p.print(x.Rparen, token.RPAREN)
+
+	case *ast.CompositeLit:
+		// composite literal elements that are composite literals themselves may have the type omitted
+		if x.Type != nil {
+			p.expr1(x.Type, token.HighestPrec, depth, compositeLit, multiLine)
+		}
+		p.print(x.Lbrace, token.LBRACE)
+		p.exprList(x.Lbrace, x.Elts, 1, commaSep|commaTerm, multiLine, x.Rbrace)
+		// do not insert extra line breaks because of comments before
+		// the closing '}' as it might break the code if there is no
+		// trailing ','
+		p.print(noExtraLinebreak, x.Rbrace, token.RBRACE, noExtraLinebreak)
+
+	case *ast.Ellipsis:
+		p.print(token.ELLIPSIS)
+		if x.Elt != nil {
+			p.expr(x.Elt, multiLine)
+		}
+
+	case *ast.ArrayType:
+		p.print(token.LBRACK)
+		if x.Len != nil {
+			p.expr(x.Len, multiLine)
+		}
+		p.print(token.RBRACK)
+		p.expr(x.Elt, multiLine)
+
+	case *ast.StructType:
+		p.print(token.STRUCT)
+		p.fieldList(x.Fields, x.Incomplete, ctxt|structType)
+
+	case *ast.FuncType:
+		p.print(token.FUNC)
+		p.signature(x.Params, x.Results, multiLine)
+
+	case *ast.InterfaceType:
+		p.print(token.INTERFACE)
+		p.fieldList(x.Methods, x.Incomplete, ctxt)
+
+	case *ast.MapType:
+		p.print(token.MAP, token.LBRACK)
+		p.expr(x.Key, multiLine)
+		p.print(token.RBRACK)
+		p.expr(x.Value, multiLine)
+
+	case *ast.ChanType:
+		switch x.Dir {
+		case ast.SEND | ast.RECV:
+			p.print(token.CHAN)
+		case ast.RECV:
+			p.print(token.ARROW, token.CHAN)
+		case ast.SEND:
+			p.print(token.CHAN, token.ARROW)
+		}
+		p.print(blank)
+		p.expr(x.Value, multiLine)
+
+	default:
+		panic("unreachable")
+	}
+
+	return
+}
+
+
+func (p *printer) expr0(x ast.Expr, depth int, multiLine *bool) {
+	p.expr1(x, token.LowestPrec, depth, 0, multiLine)
+}
+
+
+// Sets multiLine to true if the expression spans multiple lines.
+func (p *printer) expr(x ast.Expr, multiLine *bool) {
+	const depth = 1
+	p.expr1(x, token.LowestPrec, depth, 0, multiLine)
+}
+
+
+// ----------------------------------------------------------------------------
+// Statements
+
+// Print the statement list indented, but without a newline after the last statement.
+// Extra line breaks between statements in the source are respected but at most one
+// empty line is printed between statements.
+func (p *printer) stmtList(list []ast.Stmt, _indent int, nextIsRBrace bool) {
+	// TODO(gri): fix _indent code
+	if _indent > 0 {
+		p.print(indent)
+	}
+	var multiLine bool
+	for i, s := range list {
+		// _indent == 0 only for lists of switch/select case clauses;
+		// in those cases each clause is a new section
+		p.linebreak(p.fset.Position(s.Pos()).Line, 1, ignore, i == 0 || _indent == 0 || multiLine)
+		multiLine = false
+		p.stmt(s, nextIsRBrace && i == len(list)-1, &multiLine)
+	}
+	if _indent > 0 {
+		p.print(unindent)
+	}
+}
+
+
+// block prints an *ast.BlockStmt; it always spans at least two lines.
+func (p *printer) block(s *ast.BlockStmt, indent int) {
+	p.print(s.Pos(), token.LBRACE)
+	p.stmtList(s.List, indent, true)
+	p.linebreak(p.fset.Position(s.Rbrace).Line, 1, ignore, true)
+	p.print(s.Rbrace, token.RBRACE)
+}
+
+
+func isTypeName(x ast.Expr) bool {
+	switch t := x.(type) {
+	case *ast.Ident:
+		return true
+	case *ast.SelectorExpr:
+		return isTypeName(t.X)
+	}
+	return false
+}
+
+
+func stripParens(x ast.Expr) ast.Expr {
+	if px, strip := x.(*ast.ParenExpr); strip {
+		// parentheses must not be stripped if there are any
+		// unparenthesized composite literals starting with
+		// a type name
+		ast.Inspect(px.X, func(node ast.Node) bool {
+			switch x := node.(type) {
+			case *ast.ParenExpr:
+				// parentheses protect enclosed composite literals
+				return false
+			case *ast.CompositeLit:
+				if isTypeName(x.Type) {
+					strip = false // do not strip parentheses
+				}
+				return false
+			}
+			// in all other cases, keep inspecting
+			return true
+		})
+		if strip {
+			return stripParens(px.X)
+		}
+	}
+	return x
+}
+
+
+func (p *printer) controlClause(isForStmt bool, init ast.Stmt, expr ast.Expr, post ast.Stmt) {
+	p.print(blank)
+	needsBlank := false
+	if init == nil && post == nil {
+		// no semicolons required
+		if expr != nil {
+			p.expr(stripParens(expr), ignoreMultiLine)
+			needsBlank = true
+		}
+	} else {
+		// all semicolons required
+		// (they are not separators, print them explicitly)
+		if init != nil {
+			p.stmt(init, false, ignoreMultiLine)
+		}
+		p.print(token.SEMICOLON, blank)
+		if expr != nil {
+			p.expr(stripParens(expr), ignoreMultiLine)
+			needsBlank = true
+		}
+		if isForStmt {
+			p.print(token.SEMICOLON, blank)
+			needsBlank = false
+			if post != nil {
+				p.stmt(post, false, ignoreMultiLine)
+				needsBlank = true
+			}
+		}
+	}
+	if needsBlank {
+		p.print(blank)
+	}
+}
+
+
+// Sets multiLine to true if the statements spans multiple lines.
+func (p *printer) stmt(stmt ast.Stmt, nextIsRBrace bool, multiLine *bool) {
+	p.print(stmt.Pos())
+
+	switch s := stmt.(type) {
+	case *ast.BadStmt:
+		p.print("BadStmt")
+
+	case *ast.DeclStmt:
+		p.decl(s.Decl, multiLine)
+
+	case *ast.EmptyStmt:
+		// nothing to do
+
+	case *ast.LabeledStmt:
+		// a "correcting" unindent immediately following a line break
+		// is applied before the line break if there is no comment
+		// between (see writeWhitespace)
+		p.print(unindent)
+		p.expr(s.Label, multiLine)
+		p.print(s.Colon, token.COLON, indent)
+		if e, isEmpty := s.Stmt.(*ast.EmptyStmt); isEmpty {
+			if !nextIsRBrace {
+				p.print(newline, e.Pos(), token.SEMICOLON)
+				break
+			}
+		} else {
+			p.linebreak(p.fset.Position(s.Stmt.Pos()).Line, 1, ignore, true)
+		}
+		p.stmt(s.Stmt, nextIsRBrace, multiLine)
+
+	case *ast.ExprStmt:
+		const depth = 1
+		p.expr0(s.X, depth, multiLine)
+
+	case *ast.IncDecStmt:
+		const depth = 1
+		p.expr0(s.X, depth+1, multiLine)
+		p.print(s.TokPos, s.Tok)
+
+	case *ast.AssignStmt:
+		var depth = 1
+		if len(s.Lhs) > 1 && len(s.Rhs) > 1 {
+			depth++
+		}
+		p.exprList(s.Pos(), s.Lhs, depth, commaSep, multiLine, s.TokPos)
+		p.print(blank, s.TokPos, s.Tok)
+		p.exprList(s.TokPos, s.Rhs, depth, blankStart|commaSep, multiLine, token.NoPos)
+
+	case *ast.GoStmt:
+		p.print(token.GO, blank)
+		p.expr(s.Call, multiLine)
+
+	case *ast.DeferStmt:
+		p.print(token.DEFER, blank)
+		p.expr(s.Call, multiLine)
+
+	case *ast.ReturnStmt:
+		p.print(token.RETURN)
+		if s.Results != nil {
+			p.exprList(s.Pos(), s.Results, 1, blankStart|commaSep, multiLine, token.NoPos)
+		}
+
+	case *ast.BranchStmt:
+		p.print(s.Tok)
+		if s.Label != nil {
+			p.print(blank)
+			p.expr(s.Label, multiLine)
+		}
+
+	case *ast.BlockStmt:
+		p.block(s, 1)
+		*multiLine = true
+
+	case *ast.IfStmt:
+		p.print(token.IF)
+		p.controlClause(false, s.Init, s.Cond, nil)
+		p.block(s.Body, 1)
+		*multiLine = true
+		if s.Else != nil {
+			p.print(blank, token.ELSE, blank)
+			switch s.Else.(type) {
+			case *ast.BlockStmt, *ast.IfStmt:
+				p.stmt(s.Else, nextIsRBrace, ignoreMultiLine)
+			default:
+				p.print(token.LBRACE, indent, formfeed)
+				p.stmt(s.Else, true, ignoreMultiLine)
+				p.print(unindent, formfeed, token.RBRACE)
+			}
+		}
+
+	case *ast.CaseClause:
+		if s.Values != nil {
+			p.print(token.CASE)
+			p.exprList(s.Pos(), s.Values, 1, blankStart|commaSep, multiLine, s.Colon)
+		} else {
+			p.print(token.DEFAULT)
+		}
+		p.print(s.Colon, token.COLON)
+		p.stmtList(s.Body, 1, nextIsRBrace)
+
+	case *ast.SwitchStmt:
+		p.print(token.SWITCH)
+		p.controlClause(false, s.Init, s.Tag, nil)
+		p.block(s.Body, 0)
+		*multiLine = true
+
+	case *ast.TypeCaseClause:
+		if s.Types != nil {
+			p.print(token.CASE)
+			p.exprList(s.Pos(), s.Types, 1, blankStart|commaSep, multiLine, s.Colon)
+		} else {
+			p.print(token.DEFAULT)
+		}
+		p.print(s.Colon, token.COLON)
+		p.stmtList(s.Body, 1, nextIsRBrace)
+
+	case *ast.TypeSwitchStmt:
+		p.print(token.SWITCH)
+		if s.Init != nil {
+			p.print(blank)
+			p.stmt(s.Init, false, ignoreMultiLine)
+			p.print(token.SEMICOLON)
+		}
+		p.print(blank)
+		p.stmt(s.Assign, false, ignoreMultiLine)
+		p.print(blank)
+		p.block(s.Body, 0)
+		*multiLine = true
+
+	case *ast.CommClause:
+		if s.Rhs != nil {
+			p.print(token.CASE, blank)
+			if s.Lhs != nil {
+				p.expr(s.Lhs, multiLine)
+				p.print(blank, s.Tok, blank)
+			}
+			p.expr(s.Rhs, multiLine)
+		} else {
+			p.print(token.DEFAULT)
+		}
+		p.print(s.Colon, token.COLON)
+		p.stmtList(s.Body, 1, nextIsRBrace)
+
+	case *ast.SelectStmt:
+		p.print(token.SELECT, blank)
+		p.block(s.Body, 0)
+		*multiLine = true
+
+	case *ast.ForStmt:
+		p.print(token.FOR)
+		p.controlClause(true, s.Init, s.Cond, s.Post)
+		p.block(s.Body, 1)
+		*multiLine = true
+
+	case *ast.RangeStmt:
+		p.print(token.FOR, blank)
+		p.expr(s.Key, multiLine)
+		if s.Value != nil {
+			p.print(token.COMMA, blank)
+			p.expr(s.Value, multiLine)
+		}
+		p.print(blank, s.TokPos, s.Tok, blank, token.RANGE, blank)
+		p.expr(stripParens(s.X), multiLine)
+		p.print(blank)
+		p.block(s.Body, 1)
+		*multiLine = true
+
+	default:
+		panic("unreachable")
+	}
+
+	return
+}
+
+
+// ----------------------------------------------------------------------------
+// Declarations
+
+// The parameter n is the number of specs in the group. If doIndent is set,
+// multi-line identifier lists in the spec are indented when the first
+// linebreak is encountered.
+// Sets multiLine to true if the spec spans multiple lines.
+//
+func (p *printer) spec(spec ast.Spec, n int, doIndent bool, multiLine *bool) {
+	switch s := spec.(type) {
+	case *ast.ImportSpec:
+		p.setComment(s.Doc)
+		if s.Name != nil {
+			p.expr(s.Name, multiLine)
+			p.print(vtab)
+		}
+		p.expr(s.Path, multiLine)
+		p.setComment(s.Comment)
+
+	case *ast.ValueSpec:
+		p.setComment(s.Doc)
+		p.identList(s.Names, doIndent, multiLine) // always present
+		if n == 1 {
+			if s.Type != nil {
+				p.print(blank)
+				p.expr(s.Type, multiLine)
+			}
+			if s.Values != nil {
+				p.print(blank, token.ASSIGN)
+				p.exprList(token.NoPos, s.Values, 1, blankStart|commaSep, multiLine, token.NoPos)
+			}
+			p.setComment(s.Comment)
+
+		} else {
+			extraTabs := 3
+			if s.Type != nil {
+				p.print(vtab)
+				p.expr(s.Type, multiLine)
+				extraTabs--
+			}
+			if s.Values != nil {
+				p.print(vtab, token.ASSIGN)
+				p.exprList(token.NoPos, s.Values, 1, blankStart|commaSep, multiLine, token.NoPos)
+				extraTabs--
+			}
+			if s.Comment != nil {
+				for ; extraTabs > 0; extraTabs-- {
+					p.print(vtab)
+				}
+				p.setComment(s.Comment)
+			}
+		}
+
+	case *ast.TypeSpec:
+		p.setComment(s.Doc)
+		p.expr(s.Name, multiLine)
+		if n == 1 {
+			p.print(blank)
+		} else {
+			p.print(vtab)
+		}
+		p.expr(s.Type, multiLine)
+		p.setComment(s.Comment)
+
+	default:
+		panic("unreachable")
+	}
+}
+
+
+// Sets multiLine to true if the declaration spans multiple lines.
+func (p *printer) genDecl(d *ast.GenDecl, multiLine *bool) {
+	p.setComment(d.Doc)
+	p.print(d.Pos(), d.Tok, blank)
+
+	if d.Lparen.IsValid() {
+		// group of parenthesized declarations
+		p.print(d.Lparen, token.LPAREN)
+		if len(d.Specs) > 0 {
+			p.print(indent, formfeed)
+			var ml bool
+			for i, s := range d.Specs {
+				if i > 0 {
+					p.linebreak(p.fset.Position(s.Pos()).Line, 1, ignore, ml)
+				}
+				ml = false
+				p.spec(s, len(d.Specs), false, &ml)
+			}
+			p.print(unindent, formfeed)
+			*multiLine = true
+		}
+		p.print(d.Rparen, token.RPAREN)
+
+	} else {
+		// single declaration
+		p.spec(d.Specs[0], 1, true, multiLine)
+	}
+}
+
+
+// nodeSize determines the size of n in chars after formatting.
+// The result is <= maxSize if the node fits on one line with at
+// most maxSize chars and the formatted output doesn't contain
+// any control chars. Otherwise, the result is > maxSize.
+//
+func (p *printer) nodeSize(n ast.Node, maxSize int) (size int) {
+	size = maxSize + 1 // assume n doesn't fit
+	// nodeSize computation must be indendent of particular
+	// style so that we always get the same decision; print
+	// in RawFormat
+	cfg := Config{Mode: RawFormat}
+	var buf bytes.Buffer
+	if _, err := cfg.Fprint(&buf, p.fset, n); err != nil {
+		return
+	}
+	if buf.Len() <= maxSize {
+		for _, ch := range buf.Bytes() {
+			if ch < ' ' {
+				return
+			}
+		}
+		size = buf.Len() // n fits
+	}
+	return
+}
+
+
+func (p *printer) isOneLineFunc(b *ast.BlockStmt, headerSize int) bool {
+	pos1 := b.Pos()
+	pos2 := b.Rbrace
+	if pos1.IsValid() && pos2.IsValid() && p.fset.Position(pos1).Line != p.fset.Position(pos2).Line {
+		// opening and closing brace are on different lines - don't make it a one-liner
+		return false
+	}
+	if len(b.List) > 5 || p.commentBefore(p.fset.Position(pos2)) {
+		// too many statements or there is a comment inside - don't make it a one-liner
+		return false
+	}
+	// otherwise, estimate body size
+	const maxSize = 100
+	bodySize := 0
+	for i, s := range b.List {
+		if i > 0 {
+			bodySize += 2 // space for a semicolon and blank
+		}
+		bodySize += p.nodeSize(s, maxSize)
+	}
+	return headerSize+bodySize <= maxSize
+}
+
+
+// Sets multiLine to true if the function body spans multiple lines.
+func (p *printer) funcBody(b *ast.BlockStmt, headerSize int, isLit bool, multiLine *bool) {
+	if b == nil {
+		return
+	}
+
+	p.nesting++
+	defer func() {
+		p.nesting--
+	}()
+
+	if p.isOneLineFunc(b, headerSize) {
+		sep := vtab
+		if isLit {
+			sep = blank
+		}
+		p.print(sep, b.Lbrace, token.LBRACE)
+		if len(b.List) > 0 {
+			p.print(blank)
+			for i, s := range b.List {
+				if i > 0 {
+					p.print(token.SEMICOLON, blank)
+				}
+				p.stmt(s, i == len(b.List)-1, ignoreMultiLine)
+			}
+			p.print(blank)
+		}
+		p.print(b.Rbrace, token.RBRACE)
+		return
+	}
+
+	p.print(blank)
+	p.block(b, 1)
+	*multiLine = true
+}
+
+
+// distance returns the column difference between from and to if both
+// are on the same line; if they are on different lines (or unknown)
+// the result is infinity.
+func (p *printer) distance(from0 token.Pos, to token.Position) int {
+	from := p.fset.Position(from0)
+	if from.IsValid() && to.IsValid() && from.Line == to.Line {
+		return to.Column - from.Column
+	}
+	return infinity
+}
+
+
+// Sets multiLine to true if the declaration spans multiple lines.
+func (p *printer) funcDecl(d *ast.FuncDecl, multiLine *bool) {
+	p.setComment(d.Doc)
+	p.print(d.Pos(), token.FUNC, blank)
+	if d.Recv != nil {
+		p.parameters(d.Recv, multiLine) // method: print receiver
+		p.print(blank)
+	}
+	p.expr(d.Name, multiLine)
+	p.signature(d.Type.Params, d.Type.Results, multiLine)
+	p.funcBody(d.Body, p.distance(d.Pos(), p.pos), false, multiLine)
+}
+
+
+// Sets multiLine to true if the declaration spans multiple lines.
+func (p *printer) decl(decl ast.Decl, multiLine *bool) {
+	switch d := decl.(type) {
+	case *ast.BadDecl:
+		p.print(d.Pos(), "BadDecl")
+	case *ast.GenDecl:
+		p.genDecl(d, multiLine)
+	case *ast.FuncDecl:
+		p.funcDecl(d, multiLine)
+	default:
+		panic("unreachable")
+	}
+}
+
+
+// ----------------------------------------------------------------------------
+// Files
+
+func declToken(decl ast.Decl) (tok token.Token) {
+	tok = token.ILLEGAL
+	switch d := decl.(type) {
+	case *ast.GenDecl:
+		tok = d.Tok
+	case *ast.FuncDecl:
+		tok = token.FUNC
+	}
+	return
+}
+
+
+func (p *printer) file(src *ast.File) {
+	p.setComment(src.Doc)
+	p.print(src.Pos(), token.PACKAGE, blank)
+	p.expr(src.Name, ignoreMultiLine)
+
+	if len(src.Decls) > 0 {
+		tok := token.ILLEGAL
+		for _, d := range src.Decls {
+			prev := tok
+			tok = declToken(d)
+			// if the declaration token changed (e.g., from CONST to TYPE)
+			// print an empty line between top-level declarations
+			min := 1
+			if prev != tok {
+				min = 2
+			}
+			p.linebreak(p.fset.Position(d.Pos()).Line, min, ignore, false)
+			p.decl(d, ignoreMultiLine)
+		}
+	}
+
+	p.print(newline)
+}
diff --git a/libgo/go/go/printer/printer.go b/libgo/go/go/printer/printer.go
new file mode 100644
index 000000000..34b0c4e2d
--- /dev/null
+++ b/libgo/go/go/printer/printer.go
@@ -0,0 +1,1140 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// The printer package implements printing of AST nodes.
+package printer
+
+import (
+	"bytes"
+	"fmt"
+	"go/ast"
+	"go/token"
+	"io"
+	"os"
+	"path"
+	"runtime"
+	"tabwriter"
+)
+
+
+const debug = false // enable for debugging
+
+
+type whiteSpace int
+
+const (
+	ignore   = whiteSpace(0)
+	blank    = whiteSpace(' ')
+	vtab     = whiteSpace('\v')
+	newline  = whiteSpace('\n')
+	formfeed = whiteSpace('\f')
+	indent   = whiteSpace('>')
+	unindent = whiteSpace('<')
+)
+
+
+var (
+	esc       = []byte{tabwriter.Escape}
+	htab      = []byte{'\t'}
+	htabs     = []byte("\t\t\t\t\t\t\t\t")
+	newlines  = []byte("\n\n\n\n\n\n\n\n") // more than the max determined by nlines
+	formfeeds = []byte("\f\f\f\f\f\f\f\f") // more than the max determined by nlines
+
+	esc_quot = []byte(""") // shorter than """
+	esc_apos = []byte("'") // shorter than "'"
+	esc_amp  = []byte("&")
+	esc_lt   = []byte("<")
+	esc_gt   = []byte(">")
+)
+
+
+// Special positions
+var noPos token.Position // use noPos when a position is needed but not known
+var infinity = 1 << 30
+
+
+// Use ignoreMultiLine if the multiLine information is not important.
+var ignoreMultiLine = new(bool)
+
+
+// A pmode value represents the current printer mode.
+type pmode int
+
+const (
+	inLiteral pmode = 1 << iota
+	noExtraLinebreak
+)
+
+
+type printer struct {
+	// Configuration (does not change after initialization)
+	output io.Writer
+	Config
+	fset   *token.FileSet
+	errors chan os.Error
+
+	// Current state
+	nesting int         // nesting level (0: top-level (package scope), >0: functions/decls.)
+	written int         // number of bytes written
+	indent  int         // current indentation
+	mode    pmode       // current printer mode
+	lastTok token.Token // the last token printed (token.ILLEGAL if it's whitespace)
+
+	// Buffered whitespace
+	buffer []whiteSpace
+
+	// The (possibly estimated) position in the generated output;
+	// in AST space (i.e., pos is set whenever a token position is
+	// known accurately, and updated dependending on what has been
+	// written).
+	pos token.Position
+
+	// The value of pos immediately after the last item has been
+	// written using writeItem.
+	last token.Position
+
+	// HTML support
+	lastTaggedLine int // last line for which a line tag was written
+
+	// The list of all source comments, in order of appearance.
+	comments        []*ast.CommentGroup // may be nil
+	cindex          int                 // current comment index
+	useNodeComments bool                // if not set, ignore lead and line comments of nodes
+}
+
+
+func (p *printer) init(output io.Writer, cfg *Config, fset *token.FileSet) {
+	p.output = output
+	p.Config = *cfg
+	p.fset = fset
+	p.errors = make(chan os.Error)
+	p.buffer = make([]whiteSpace, 0, 16) // whitespace sequences are short
+}
+
+
+func (p *printer) internalError(msg ...interface{}) {
+	if debug {
+		fmt.Print(p.pos.String() + ": ")
+		fmt.Println(msg...)
+		panic("go/printer")
+	}
+}
+
+
+// nlines returns the adjusted number of linebreaks given the desired number
+// of breaks n such that min <= result <= max where max depends on the current
+// nesting level.
+//
+func (p *printer) nlines(n, min int) int {
+	if n < min {
+		return min
+	}
+	max := 3 // max. number of newlines at the top level (p.nesting == 0)
+	if p.nesting > 0 {
+		max = 2 // max. number of newlines everywhere else
+	}
+	if n > max {
+		return max
+	}
+	return n
+}
+
+
+// write0 writes raw (uninterpreted) data to p.output and handles errors.
+// write0 does not indent after newlines, and does not HTML-escape or update p.pos.
+//
+func (p *printer) write0(data []byte) {
+	n, err := p.output.Write(data)
+	p.written += n
+	if err != nil {
+		p.errors <- err
+		runtime.Goexit()
+	}
+}
+
+
+// write interprets data and writes it to p.output. It inserts indentation
+// after a line break unless in a tabwriter escape sequence, and it HTML-
+// escapes characters if GenHTML is set. It updates p.pos as a side-effect.
+//
+func (p *printer) write(data []byte) {
+	i0 := 0
+	for i, b := range data {
+		switch b {
+		case '\n', '\f':
+			// write segment ending in b
+			p.write0(data[i0 : i+1])
+
+			// update p.pos
+			p.pos.Offset += i + 1 - i0
+			p.pos.Line++
+			p.pos.Column = 1
+
+			if p.mode&inLiteral == 0 {
+				// write indentation
+				// use "hard" htabs - indentation columns
+				// must not be discarded by the tabwriter
+				j := p.indent
+				for ; j > len(htabs); j -= len(htabs) {
+					p.write0(htabs)
+				}
+				p.write0(htabs[0:j])
+
+				// update p.pos
+				p.pos.Offset += p.indent
+				p.pos.Column += p.indent
+			}
+
+			// next segment start
+			i0 = i + 1
+
+		case '"', '\'', '&', '<', '>':
+			if p.Mode&GenHTML != 0 {
+				// write segment ending in b
+				p.write0(data[i0:i])
+
+				// write HTML-escaped b
+				var esc []byte
+				switch b {
+				case '"':
+					esc = esc_quot
+				case '\'':
+					esc = esc_apos
+				case '&':
+					esc = esc_amp
+				case '<':
+					esc = esc_lt
+				case '>':
+					esc = esc_gt
+				}
+				p.write0(esc)
+
+				// update p.pos
+				d := i + 1 - i0
+				p.pos.Offset += d
+				p.pos.Column += d
+
+				// next segment start
+				i0 = i + 1
+			}
+
+		case tabwriter.Escape:
+			p.mode ^= inLiteral
+
+			// ignore escape chars introduced by printer - they are
+			// invisible and must not affect p.pos (was issue #1089)
+			p.pos.Offset--
+			p.pos.Column--
+		}
+	}
+
+	// write remaining segment
+	p.write0(data[i0:])
+
+	// update p.pos
+	d := len(data) - i0
+	p.pos.Offset += d
+	p.pos.Column += d
+}
+
+
+func (p *printer) writeNewlines(n int, useFF bool) {
+	if n > 0 {
+		n = p.nlines(n, 0)
+		if useFF {
+			p.write(formfeeds[0:n])
+		} else {
+			p.write(newlines[0:n])
+		}
+	}
+}
+
+
+func (p *printer) writeTaggedItem(data []byte, tag HTMLTag) {
+	// write start tag, if any
+	// (no html-escaping and no p.pos update for tags - use write0)
+	if tag.Start != "" {
+		p.write0([]byte(tag.Start))
+	}
+	p.write(data)
+	// write end tag, if any
+	if tag.End != "" {
+		p.write0([]byte(tag.End))
+	}
+}
+
+
+// writeItem writes data at position pos. data is the text corresponding to
+// a single lexical token, but may also be comment text. pos is the actual
+// (or at least very accurately estimated) position of the data in the original
+// source text. If tags are present and GenHTML is set, the tags are written
+// before and after the data. writeItem updates p.last to the position
+// immediately following the data.
+//
+func (p *printer) writeItem(pos token.Position, data []byte, tag HTMLTag) {
+	fileChanged := false
+	if pos.IsValid() {
+		// continue with previous position if we don't have a valid pos
+		if p.last.IsValid() && p.last.Filename != pos.Filename {
+			// the file has changed - reset state
+			// (used when printing merged ASTs of different files
+			// e.g., the result of ast.MergePackageFiles)
+			p.indent = 0
+			p.mode = 0
+			p.buffer = p.buffer[0:0]
+			fileChanged = true
+		}
+		p.pos = pos
+	}
+	if debug {
+		// do not update p.pos - use write0
+		_, filename := path.Split(pos.Filename)
+		p.write0([]byte(fmt.Sprintf("[%s:%d:%d]", filename, pos.Line, pos.Column)))
+	}
+	if p.Mode&GenHTML != 0 {
+		// write line tag if on a new line
+		// TODO(gri): should write line tags on each line at the start
+		//            will be more useful (e.g. to show line numbers)
+		if p.Styler != nil && (pos.Line != p.lastTaggedLine || fileChanged) {
+			p.writeTaggedItem(p.Styler.LineTag(pos.Line))
+			p.lastTaggedLine = pos.Line
+		}
+		p.writeTaggedItem(data, tag)
+	} else {
+		p.write(data)
+	}
+	p.last = p.pos
+}
+
+
+// writeCommentPrefix writes the whitespace before a comment.
+// If there is any pending whitespace, it consumes as much of
+// it as is likely to help position the comment nicely.
+// pos is the comment position, next the position of the item
+// after all pending comments, isFirst indicates if this is the
+// first comment in a group of comments, and isKeyword indicates
+// if the next item is a keyword.
+//
+func (p *printer) writeCommentPrefix(pos, next token.Position, isFirst, isKeyword bool) {
+	if !p.last.IsValid() {
+		// there was no preceeding item and the comment is the
+		// first item to be printed - don't write any whitespace
+		return
+	}
+
+	if pos.IsValid() && pos.Filename != p.last.Filename {
+		// comment in a different file - separate with newlines (writeNewlines will limit the number)
+		p.writeNewlines(10, true)
+		return
+	}
+
+	if pos.IsValid() && pos.Line == p.last.Line {
+		// comment on the same line as last item:
+		// separate with at least one separator
+		hasSep := false
+		if isFirst {
+			j := 0
+			for i, ch := range p.buffer {
+				switch ch {
+				case blank:
+					// ignore any blanks before a comment
+					p.buffer[i] = ignore
+					continue
+				case vtab:
+					// respect existing tabs - important
+					// for proper formatting of commented structs
+					hasSep = true
+					continue
+				case indent:
+					// apply pending indentation
+					continue
+				}
+				j = i
+				break
+			}
+			p.writeWhitespace(j)
+		}
+		// make sure there is at least one separator
+		if !hasSep {
+			if pos.Line == next.Line {
+				// next item is on the same line as the comment
+				// (which must be a /*-style comment): separate
+				// with a blank instead of a tab
+				p.write([]byte{' '})
+			} else {
+				p.write(htab)
+			}
+		}
+
+	} else {
+		// comment on a different line:
+		// separate with at least one line break
+		if isFirst {
+			j := 0
+			for i, ch := range p.buffer {
+				switch ch {
+				case blank, vtab:
+					// ignore any horizontal whitespace before line breaks
+					p.buffer[i] = ignore
+					continue
+				case indent:
+					// apply pending indentation
+					continue
+				case unindent:
+					// if the next token is a keyword, apply the outdent
+					// if it appears that the comment is aligned with the
+					// keyword; otherwise assume the outdent is part of a
+					// closing block and stop (this scenario appears with
+					// comments before a case label where the comments
+					// apply to the next case instead of the current one)
+					if isKeyword && pos.Column == next.Column {
+						continue
+					}
+				case newline, formfeed:
+					// TODO(gri): may want to keep formfeed info in some cases
+					p.buffer[i] = ignore
+				}
+				j = i
+				break
+			}
+			p.writeWhitespace(j)
+		}
+		// use formfeeds to break columns before a comment;
+		// this is analogous to using formfeeds to separate
+		// individual lines of /*-style comments
+		// (if !pos.IsValid(), pos.Line == 0, and this will
+		// print no newlines)
+		p.writeNewlines(pos.Line-p.last.Line, true)
+	}
+}
+
+
+func (p *printer) writeCommentLine(comment *ast.Comment, pos token.Position, line []byte) {
+	// line must pass through unchanged, bracket it with tabwriter.Escape
+	line = bytes.Join([][]byte{esc, line, esc}, nil)
+
+	// apply styler, if any
+	var tag HTMLTag
+	if p.Styler != nil {
+		line, tag = p.Styler.Comment(comment, line)
+	}
+
+	p.writeItem(pos, line, tag)
+}
+
+
+// TODO(gri): Similar (but not quite identical) functionality for
+//            comment processing can be found in go/doc/comment.go.
+//            Perhaps this can be factored eventually.
+
+// Split comment text into lines
+func split(text []byte) [][]byte {
+	// count lines (comment text never ends in a newline)
+	n := 1
+	for _, c := range text {
+		if c == '\n' {
+			n++
+		}
+	}
+
+	// split
+	lines := make([][]byte, n)
+	n = 0
+	i := 0
+	for j, c := range text {
+		if c == '\n' {
+			lines[n] = text[i:j] // exclude newline
+			i = j + 1            // discard newline
+			n++
+		}
+	}
+	lines[n] = text[i:]
+
+	return lines
+}
+
+
+func isBlank(s []byte) bool {
+	for _, b := range s {
+		if b > ' ' {
+			return false
+		}
+	}
+	return true
+}
+
+
+func commonPrefix(a, b []byte) []byte {
+	i := 0
+	for i < len(a) && i < len(b) && a[i] == b[i] && (a[i] <= ' ' || a[i] == '*') {
+		i++
+	}
+	return a[0:i]
+}
+
+
+func stripCommonPrefix(lines [][]byte) {
+	if len(lines) < 2 {
+		return // at most one line - nothing to do
+	}
+	// len(lines) >= 2
+
+	// The heuristic in this function tries to handle a few
+	// common patterns of /*-style comments: Comments where
+	// the opening /* and closing */ are aligned and the
+	// rest of the comment text is aligned and indented with
+	// blanks or tabs, cases with a vertical "line of stars"
+	// on the left, and cases where the closing */ is on the
+	// same line as the last comment text.
+
+	// Compute maximum common white prefix of all but the first,
+	// last, and blank lines, and replace blank lines with empty
+	// lines (the first line starts with /* and has no prefix).
+	// In case of two-line comments, consider the last line for
+	// the prefix computation since otherwise the prefix would
+	// be empty.
+	//
+	// Note that the first and last line are never empty (they
+	// contain the opening /* and closing */ respectively) and
+	// thus they can be ignored by the blank line check.
+	var prefix []byte
+	if len(lines) > 2 {
+		for i, line := range lines[1 : len(lines)-1] {
+			switch {
+			case isBlank(line):
+				lines[1+i] = nil // range starts at line 1
+			case prefix == nil:
+				prefix = commonPrefix(line, line)
+			default:
+				prefix = commonPrefix(prefix, line)
+			}
+		}
+	} else { // len(lines) == 2
+		line := lines[1]
+		prefix = commonPrefix(line, line)
+	}
+
+	/*
+	 * Check for vertical "line of stars" and correct prefix accordingly.
+	 */
+	lineOfStars := false
+	if i := bytes.Index(prefix, []byte{'*'}); i >= 0 {
+		// Line of stars present.
+		if i > 0 && prefix[i-1] == ' ' {
+			i-- // remove trailing blank from prefix so stars remain aligned
+		}
+		prefix = prefix[0:i]
+		lineOfStars = true
+	} else {
+		// No line of stars present.
+		// Determine the white space on the first line after the /*
+		// and before the beginning of the comment text, assume two
+		// blanks instead of the /* unless the first character after
+		// the /* is a tab. If the first comment line is empty but
+		// for the opening /*, assume up to 3 blanks or a tab. This
+		// whitespace may be found as suffix in the common prefix.
+		first := lines[0]
+		if isBlank(first[2:]) {
+			// no comment text on the first line:
+			// reduce prefix by up to 3 blanks or a tab
+			// if present - this keeps comment text indented
+			// relative to the /* and */'s if it was indented
+			// in the first place
+			i := len(prefix)
+			for n := 0; n < 3 && i > 0 && prefix[i-1] == ' '; n++ {
+				i--
+			}
+			if i == len(prefix) && i > 0 && prefix[i-1] == '\t' {
+				i--
+			}
+			prefix = prefix[0:i]
+		} else {
+			// comment text on the first line
+			suffix := make([]byte, len(first))
+			n := 2 // start after opening /*
+			for n < len(first) && first[n] <= ' ' {
+				suffix[n] = first[n]
+				n++
+			}
+			if n > 2 && suffix[2] == '\t' {
+				// assume the '\t' compensates for the /*
+				suffix = suffix[2:n]
+			} else {
+				// otherwise assume two blanks
+				suffix[0], suffix[1] = ' ', ' '
+				suffix = suffix[0:n]
+			}
+			// Shorten the computed common prefix by the length of
+			// suffix, if it is found as suffix of the prefix.
+			if bytes.HasSuffix(prefix, suffix) {
+				prefix = prefix[0 : len(prefix)-len(suffix)]
+			}
+		}
+	}
+
+	// Handle last line: If it only contains a closing */, align it
+	// with the opening /*, otherwise align the text with the other
+	// lines.
+	last := lines[len(lines)-1]
+	closing := []byte("*/")
+	i := bytes.Index(last, closing)
+	if isBlank(last[0:i]) {
+		// last line only contains closing */
+		var sep []byte
+		if lineOfStars {
+			// insert an aligning blank
+			sep = []byte{' '}
+		}
+		lines[len(lines)-1] = bytes.Join([][]byte{prefix, closing}, sep)
+	} else {
+		// last line contains more comment text - assume
+		// it is aligned like the other lines
+		prefix = commonPrefix(prefix, last)
+	}
+
+	// Remove the common prefix from all but the first and empty lines.
+	for i, line := range lines[1:] {
+		if len(line) != 0 {
+			lines[1+i] = line[len(prefix):] // range starts at line 1
+		}
+	}
+}
+
+
+func (p *printer) writeComment(comment *ast.Comment) {
+	text := comment.Text
+
+	// shortcut common case of //-style comments
+	if text[1] == '/' {
+		p.writeCommentLine(comment, p.fset.Position(comment.Pos()), text)
+		return
+	}
+
+	// for /*-style comments, print line by line and let the
+	// write function take care of the proper indentation
+	lines := split(text)
+	stripCommonPrefix(lines)
+
+	// write comment lines, separated by formfeed,
+	// without a line break after the last line
+	linebreak := formfeeds[0:1]
+	pos := p.fset.Position(comment.Pos())
+	for i, line := range lines {
+		if i > 0 {
+			p.write(linebreak)
+			pos = p.pos
+		}
+		if len(line) > 0 {
+			p.writeCommentLine(comment, pos, line)
+		}
+	}
+}
+
+
+// writeCommentSuffix writes a line break after a comment if indicated
+// and processes any leftover indentation information. If a line break
+// is needed, the kind of break (newline vs formfeed) depends on the
+// pending whitespace. writeCommentSuffix returns true if a pending
+// formfeed was dropped from the whitespace buffer.
+//
+func (p *printer) writeCommentSuffix(needsLinebreak bool) (droppedFF bool) {
+	for i, ch := range p.buffer {
+		switch ch {
+		case blank, vtab:
+			// ignore trailing whitespace
+			p.buffer[i] = ignore
+		case indent, unindent:
+			// don't loose indentation information
+		case newline, formfeed:
+			// if we need a line break, keep exactly one
+			// but remember if we dropped any formfeeds
+			if needsLinebreak {
+				needsLinebreak = false
+			} else {
+				if ch == formfeed {
+					droppedFF = true
+				}
+				p.buffer[i] = ignore
+			}
+		}
+	}
+	p.writeWhitespace(len(p.buffer))
+
+	// make sure we have a line break
+	if needsLinebreak {
+		p.write([]byte{'\n'})
+	}
+
+	return
+}
+
+
+// intersperseComments consumes all comments that appear before the next token
+// tok and prints it together with the buffered whitespace (i.e., the whitespace
+// that needs to be written before the next token). A heuristic is used to mix
+// the comments and whitespace. intersperseComments returns true if a pending
+// formfeed was dropped from the whitespace buffer.
+//
+func (p *printer) intersperseComments(next token.Position, tok token.Token) (droppedFF bool) {
+	var last *ast.Comment
+	for ; p.commentBefore(next); p.cindex++ {
+		for _, c := range p.comments[p.cindex].List {
+			p.writeCommentPrefix(p.fset.Position(c.Pos()), next, last == nil, tok.IsKeyword())
+			p.writeComment(c)
+			last = c
+		}
+	}
+
+	if last != nil {
+		if last.Text[1] == '*' && p.fset.Position(last.Pos()).Line == next.Line {
+			// the last comment is a /*-style comment and the next item
+			// follows on the same line: separate with an extra blank
+			p.write([]byte{' '})
+		}
+		// ensure that there is a line break after a //-style comment,
+		// before a closing '}' unless explicitly disabled, or at eof
+		needsLinebreak :=
+			last.Text[1] == '/' ||
+				tok == token.RBRACE && p.mode&noExtraLinebreak == 0 ||
+				tok == token.EOF
+		return p.writeCommentSuffix(needsLinebreak)
+	}
+
+	// no comment was written - we should never reach here since
+	// intersperseComments should not be called in that case
+	p.internalError("intersperseComments called without pending comments")
+	return false
+}
+
+
+// whiteWhitespace writes the first n whitespace entries.
+func (p *printer) writeWhitespace(n int) {
+	// write entries
+	var data [1]byte
+	for i := 0; i < n; i++ {
+		switch ch := p.buffer[i]; ch {
+		case ignore:
+			// ignore!
+		case indent:
+			p.indent++
+		case unindent:
+			p.indent--
+			if p.indent < 0 {
+				p.internalError("negative indentation:", p.indent)
+				p.indent = 0
+			}
+		case newline, formfeed:
+			// A line break immediately followed by a "correcting"
+			// unindent is swapped with the unindent - this permits
+			// proper label positioning. If a comment is between
+			// the line break and the label, the unindent is not
+			// part of the comment whitespace prefix and the comment
+			// will be positioned correctly indented.
+			if i+1 < n && p.buffer[i+1] == unindent {
+				// Use a formfeed to terminate the current section.
+				// Otherwise, a long label name on the next line leading
+				// to a wide column may increase the indentation column
+				// of lines before the label; effectively leading to wrong
+				// indentation.
+				p.buffer[i], p.buffer[i+1] = unindent, formfeed
+				i-- // do it again
+				continue
+			}
+			fallthrough
+		default:
+			data[0] = byte(ch)
+			p.write(data[0:])
+		}
+	}
+
+	// shift remaining entries down
+	i := 0
+	for ; n < len(p.buffer); n++ {
+		p.buffer[i] = p.buffer[n]
+		i++
+	}
+	p.buffer = p.buffer[0:i]
+}
+
+
+// ----------------------------------------------------------------------------
+// Printing interface
+
+
+func mayCombine(prev token.Token, next byte) (b bool) {
+	switch prev {
+	case token.INT:
+		b = next == '.' // 1.
+	case token.ADD:
+		b = next == '+' // ++
+	case token.SUB:
+		b = next == '-' // --
+	case token.QUO:
+		b = next == '*' // /*
+	case token.LSS:
+		b = next == '-' || next == '<' // <- or <<
+	case token.AND:
+		b = next == '&' || next == '^' // && or &^
+	}
+	return
+}
+
+
+// print prints a list of "items" (roughly corresponding to syntactic
+// tokens, but also including whitespace and formatting information).
+// It is the only print function that should be called directly from
+// any of the AST printing functions in nodes.go.
+//
+// Whitespace is accumulated until a non-whitespace token appears. Any
+// comments that need to appear before that token are printed first,
+// taking into account the amount and structure of any pending white-
+// space for best comment placement. Then, any leftover whitespace is
+// printed, followed by the actual token.
+//
+func (p *printer) print(args ...interface{}) {
+	for _, f := range args {
+		next := p.pos // estimated position of next item
+		var data []byte
+		var tag HTMLTag
+		var tok token.Token
+
+		switch x := f.(type) {
+		case pmode:
+			// toggle printer mode
+			p.mode ^= x
+		case whiteSpace:
+			if x == ignore {
+				// don't add ignore's to the buffer; they
+				// may screw up "correcting" unindents (see
+				// LabeledStmt)
+				break
+			}
+			i := len(p.buffer)
+			if i == cap(p.buffer) {
+				// Whitespace sequences are very short so this should
+				// never happen. Handle gracefully (but possibly with
+				// bad comment placement) if it does happen.
+				p.writeWhitespace(i)
+				i = 0
+			}
+			p.buffer = p.buffer[0 : i+1]
+			p.buffer[i] = x
+		case *ast.Ident:
+			if p.Styler != nil {
+				data, tag = p.Styler.Ident(x)
+			} else {
+				data = []byte(x.Name)
+			}
+			tok = token.IDENT
+		case *ast.BasicLit:
+			if p.Styler != nil {
+				data, tag = p.Styler.BasicLit(x)
+			} else {
+				data = x.Value
+			}
+			// escape all literals so they pass through unchanged
+			// (note that valid Go programs cannot contain
+			// tabwriter.Escape bytes since they do not appear in
+			// legal UTF-8 sequences)
+			escData := make([]byte, 0, len(data)+2)
+			escData = append(escData, tabwriter.Escape)
+			escData = append(escData, data...)
+			escData = append(escData, tabwriter.Escape)
+			data = escData
+			tok = x.Kind
+		case token.Token:
+			s := x.String()
+			if mayCombine(p.lastTok, s[0]) {
+				// the previous and the current token must be
+				// separated by a blank otherwise they combine
+				// into a different incorrect token sequence
+				// (except for token.INT followed by a '.' this
+				// should never happen because it is taken care
+				// of via binary expression formatting)
+				if len(p.buffer) != 0 {
+					p.internalError("whitespace buffer not empty")
+				}
+				p.buffer = p.buffer[0:1]
+				p.buffer[0] = ' '
+			}
+			if p.Styler != nil {
+				data, tag = p.Styler.Token(x)
+			} else {
+				data = []byte(s)
+			}
+			tok = x
+		case token.Pos:
+			if x.IsValid() {
+				next = p.fset.Position(x) // accurate position of next item
+			}
+			tok = p.lastTok
+		default:
+			fmt.Fprintf(os.Stderr, "print: unsupported argument type %T\n", f)
+			panic("go/printer type")
+		}
+		p.lastTok = tok
+		p.pos = next
+
+		if data != nil {
+			droppedFF := p.flush(next, tok)
+
+			// intersperse extra newlines if present in the source
+			// (don't do this in flush as it will cause extra newlines
+			// at the end of a file) - use formfeeds if we dropped one
+			// before
+			p.writeNewlines(next.Line-p.pos.Line, droppedFF)
+
+			p.writeItem(next, data, tag)
+		}
+	}
+}
+
+
+// commentBefore returns true iff the current comment occurs
+// before the next position in the source code.
+//
+func (p *printer) commentBefore(next token.Position) bool {
+	return p.cindex < len(p.comments) && p.fset.Position(p.comments[p.cindex].List[0].Pos()).Offset < next.Offset
+}
+
+
+// Flush prints any pending comments and whitespace occurring
+// textually before the position of the next token tok. Flush
+// returns true if a pending formfeed character was dropped
+// from the whitespace buffer as a result of interspersing
+// comments.
+//
+func (p *printer) flush(next token.Position, tok token.Token) (droppedFF bool) {
+	if p.commentBefore(next) {
+		// if there are comments before the next item, intersperse them
+		droppedFF = p.intersperseComments(next, tok)
+	} else {
+		// otherwise, write any leftover whitespace
+		p.writeWhitespace(len(p.buffer))
+	}
+	return
+}
+
+
+// ----------------------------------------------------------------------------
+// Trimmer
+
+// A trimmer is an io.Writer filter for stripping tabwriter.Escape
+// characters, trailing blanks and tabs, and for converting formfeed
+// and vtab characters into newlines and htabs (in case no tabwriter
+// is used). Text bracketed by tabwriter.Escape characters is passed
+// through unchanged.
+//
+type trimmer struct {
+	output io.Writer
+	space  bytes.Buffer
+	state  int
+}
+
+
+// trimmer is implemented as a state machine.
+// It can be in one of the following states:
+const (
+	inSpace = iota
+	inEscape
+	inText
+)
+
+
+// Design note: It is tempting to eliminate extra blanks occurring in
+//              whitespace in this function as it could simplify some
+//              of the blanks logic in the node printing functions.
+//              However, this would mess up any formatting done by
+//              the tabwriter.
+
+func (p *trimmer) Write(data []byte) (n int, err os.Error) {
+	m := 0 // if p.state != inSpace, data[m:n] is unwritten
+	var b byte
+	for n, b = range data {
+		if b == '\v' {
+			b = '\t' // convert to htab
+		}
+		switch p.state {
+		case inSpace:
+			switch b {
+			case '\t', ' ':
+				p.space.WriteByte(b) // WriteByte returns no errors
+			case '\f', '\n':
+				p.space.Reset()                        // discard trailing space
+				_, err = p.output.Write(newlines[0:1]) // write newline
+			case tabwriter.Escape:
+				_, err = p.output.Write(p.space.Bytes())
+				p.space.Reset()
+				p.state = inEscape
+				m = n + 1 // drop tabwriter.Escape
+			default:
+				_, err = p.output.Write(p.space.Bytes())
+				p.space.Reset()
+				p.state = inText
+				m = n
+			}
+		case inEscape:
+			if b == tabwriter.Escape {
+				_, err = p.output.Write(data[m:n])
+				p.state = inSpace
+			}
+		case inText:
+			switch b {
+			case '\t', ' ':
+				_, err = p.output.Write(data[m:n])
+				p.state = inSpace
+				p.space.WriteByte(b) // WriteByte returns no errors
+			case '\f':
+				data[n] = '\n' // convert to newline
+			case tabwriter.Escape:
+				_, err = p.output.Write(data[m:n])
+				p.state = inEscape
+				m = n + 1 // drop tabwriter.Escape
+			}
+		}
+		if err != nil {
+			return
+		}
+	}
+	n = len(data)
+
+	if p.state != inSpace {
+		_, err = p.output.Write(data[m:n])
+		p.state = inSpace
+	}
+
+	return
+}
+
+
+// ----------------------------------------------------------------------------
+// Public interface
+
+// General printing is controlled with these Config.Mode flags.
+const (
+	GenHTML   uint = 1 << iota // generate HTML
+	RawFormat                  // do not use a tabwriter; if set, UseSpaces is ignored
+	TabIndent                  // use tabs for indentation independent of UseSpaces
+	UseSpaces                  // use spaces instead of tabs for alignment
+)
+
+
+// An HTMLTag specifies a start and end tag.
+type HTMLTag struct {
+	Start, End string // empty if tags are absent
+}
+
+
+// A Styler specifies formatting of line tags and elementary Go words.
+// A format consists of text and a (possibly empty) surrounding HTML tag.
+//
+type Styler interface {
+	LineTag(line int) ([]byte, HTMLTag)
+	Comment(c *ast.Comment, line []byte) ([]byte, HTMLTag)
+	BasicLit(x *ast.BasicLit) ([]byte, HTMLTag)
+	Ident(id *ast.Ident) ([]byte, HTMLTag)
+	Token(tok token.Token) ([]byte, HTMLTag)
+}
+
+
+// A Config node controls the output of Fprint.
+type Config struct {
+	Mode     uint   // default: 0
+	Tabwidth int    // default: 8
+	Styler   Styler // default: nil
+}
+
+
+// Fprint "pretty-prints" an AST node to output and returns the number
+// of bytes written and an error (if any) for a given configuration cfg.
+// Position information is interpreted relative to the file set fset.
+// The node type must be *ast.File, or assignment-compatible to ast.Expr,
+// ast.Decl, ast.Spec, or ast.Stmt.
+//
+func (cfg *Config) Fprint(output io.Writer, fset *token.FileSet, node interface{}) (int, os.Error) {
+	// redirect output through a trimmer to eliminate trailing whitespace
+	// (Input to a tabwriter must be untrimmed since trailing tabs provide
+	// formatting information. The tabwriter could provide trimming
+	// functionality but no tabwriter is used when RawFormat is set.)
+	output = &trimmer{output: output}
+
+	// setup tabwriter if needed and redirect output
+	var tw *tabwriter.Writer
+	if cfg.Mode&RawFormat == 0 {
+		minwidth := cfg.Tabwidth
+
+		padchar := byte('\t')
+		if cfg.Mode&UseSpaces != 0 {
+			padchar = ' '
+		}
+
+		twmode := tabwriter.DiscardEmptyColumns
+		if cfg.Mode&GenHTML != 0 {
+			twmode |= tabwriter.FilterHTML
+		}
+		if cfg.Mode&TabIndent != 0 {
+			minwidth = 0
+			twmode |= tabwriter.TabIndent
+		}
+
+		tw = tabwriter.NewWriter(output, minwidth, cfg.Tabwidth, 1, padchar, twmode)
+		output = tw
+	}
+
+	// setup printer and print node
+	var p printer
+	p.init(output, cfg, fset)
+	go func() {
+		switch n := node.(type) {
+		case ast.Expr:
+			p.nesting = 1
+			p.useNodeComments = true
+			p.expr(n, ignoreMultiLine)
+		case ast.Stmt:
+			p.nesting = 1
+			p.useNodeComments = true
+			// A labeled statement will un-indent to position the
+			// label. Set indent to 1 so we don't get indent "underflow".
+			if _, labeledStmt := n.(*ast.LabeledStmt); labeledStmt {
+				p.indent = 1
+			}
+			p.stmt(n, false, ignoreMultiLine)
+		case ast.Decl:
+			p.nesting = 1
+			p.useNodeComments = true
+			p.decl(n, ignoreMultiLine)
+		case ast.Spec:
+			p.nesting = 1
+			p.useNodeComments = true
+			p.spec(n, 1, false, ignoreMultiLine)
+		case *ast.File:
+			p.nesting = 0
+			p.comments = n.Comments
+			p.useNodeComments = n.Comments == nil
+			p.file(n)
+		default:
+			p.errors <- fmt.Errorf("printer.Fprint: unsupported node type %T", n)
+			runtime.Goexit()
+		}
+		p.flush(token.Position{Offset: infinity, Line: infinity}, token.EOF)
+		p.errors <- nil // no errors
+	}()
+	err := <-p.errors // wait for completion of goroutine
+
+	// flush tabwriter, if any
+	if tw != nil {
+		tw.Flush() // ignore errors
+	}
+
+	return p.written, err
+}
+
+
+// Fprint "pretty-prints" an AST node to output.
+// It calls Config.Fprint with default settings.
+//
+func Fprint(output io.Writer, fset *token.FileSet, node interface{}) os.Error {
+	_, err := (&Config{Tabwidth: 8}).Fprint(output, fset, node) // don't care about number of bytes written
+	return err
+}
diff --git a/libgo/go/go/printer/printer_test.go b/libgo/go/go/printer/printer_test.go
new file mode 100644
index 000000000..c66471b92
--- /dev/null
+++ b/libgo/go/go/printer/printer_test.go
@@ -0,0 +1,138 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package printer
+
+import (
+	"bytes"
+	"flag"
+	"io/ioutil"
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"path"
+	"testing"
+)
+
+
+const (
+	dataDir  = "testdata"
+	tabwidth = 8
+)
+
+
+var update = flag.Bool("update", false, "update golden files")
+
+
+var fset = token.NewFileSet()
+
+
+func lineString(text []byte, i int) string {
+	i0 := i
+	for i < len(text) && text[i] != '\n' {
+		i++
+	}
+	return string(text[i0:i])
+}
+
+
+type checkMode uint
+
+const (
+	export checkMode = 1 << iota
+	rawFormat
+)
+
+
+func check(t *testing.T, source, golden string, mode checkMode) {
+	// parse source
+	prog, err := parser.ParseFile(fset, source, nil, parser.ParseComments)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	// filter exports if necessary
+	if mode&export != 0 {
+		ast.FileExports(prog) // ignore result
+		prog.Comments = nil   // don't print comments that are not in AST
+	}
+
+	// determine printer configuration
+	cfg := Config{Tabwidth: tabwidth}
+	if mode&rawFormat != 0 {
+		cfg.Mode |= RawFormat
+	}
+
+	// format source
+	var buf bytes.Buffer
+	if _, err := cfg.Fprint(&buf, fset, prog); err != nil {
+		t.Error(err)
+	}
+	res := buf.Bytes()
+
+	// update golden files if necessary
+	if *update {
+		if err := ioutil.WriteFile(golden, res, 0644); err != nil {
+			t.Error(err)
+		}
+		return
+	}
+
+	// get golden
+	gld, err := ioutil.ReadFile(golden)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	// compare lengths
+	if len(res) != len(gld) {
+		t.Errorf("len = %d, expected %d (= len(%s))", len(res), len(gld), golden)
+	}
+
+	// compare contents
+	for i, line, offs := 0, 1, 0; i < len(res) && i < len(gld); i++ {
+		ch := res[i]
+		if ch != gld[i] {
+			t.Errorf("%s:%d:%d: %s", source, line, i-offs+1, lineString(res, offs))
+			t.Errorf("%s:%d:%d: %s", golden, line, i-offs+1, lineString(gld, offs))
+			t.Error()
+			return
+		}
+		if ch == '\n' {
+			line++
+			offs = i + 1
+		}
+	}
+}
+
+
+type entry struct {
+	source, golden string
+	mode           checkMode
+}
+
+// Use gotest -update to create/update the respective golden files.
+var data = []entry{
+	{"empty.input", "empty.golden", 0},
+	{"comments.input", "comments.golden", 0},
+	{"comments.input", "comments.x", export},
+	{"linebreaks.input", "linebreaks.golden", 0},
+	{"expressions.input", "expressions.golden", 0},
+	{"expressions.input", "expressions.raw", rawFormat},
+	{"declarations.input", "declarations.golden", 0},
+	{"statements.input", "statements.golden", 0},
+}
+
+
+func Test(t *testing.T) {
+	for _, e := range data {
+		source := path.Join(dataDir, e.source)
+		golden := path.Join(dataDir, e.golden)
+		check(t, source, golden, e.mode)
+		// TODO(gri) check that golden is idempotent
+		//check(t, golden, golden, e.mode);
+	}
+}
diff --git a/libgo/go/go/printer/testdata/comments.golden b/libgo/go/go/printer/testdata/comments.golden
new file mode 100644
index 000000000..a86d66174
--- /dev/null
+++ b/libgo/go/go/printer/testdata/comments.golden
@@ -0,0 +1,483 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This is a package for testing comment placement by go/printer.
+//
+package main
+
+import "fmt"	// fmt
+
+const c0 = 0	// zero
+const (
+	c1	= iota	// c1
+	c2		// c2
+)
+
+// Alignment of comments in declarations>
+const (
+	_	T	= iota	// comment
+	_			// comment
+	_			// comment
+	_	= iota + 10
+	_	// comments
+
+	_	= 10		// comment
+	_	T	= 20	// comment
+)
+
+const (
+	_____	= iota	// foo
+	_		// bar
+	_	= 0	// bal
+	_		// bat
+)
+
+const (
+	_	T	= iota	// comment
+	_			// comment
+	_			// comment
+	_	= iota + 10
+	_	// comment
+	_	= 10
+	_	= 20		// comment
+	_	T	= 0	// comment
+)
+
+// The SZ struct; it is empty.
+type SZ struct{}
+
+// The S0 struct; no field is exported.
+type S0 struct {
+	int
+	x, y, z	int	// 3 unexported fields
+}
+
+// The S1 struct; some fields are not exported.
+type S1 struct {
+	S0
+	A, B, C	float	// 3 exported fields
+	D, b, c	int	// 2 unexported fields
+}
+
+// The S2 struct; all fields are exported.
+type S2 struct {
+	S1
+	A, B, C	float	// 3 exported fields
+}
+
+// The IZ interface; it is empty.
+type SZ interface{}
+
+// The I0 interface; no method is exported.
+type I0 interface {
+	f(x int) int	// unexported method
+}
+
+// The I1 interface; some methods are not exported.
+type I1 interface {
+	I0
+	F(x float) float	// exported methods
+	g(x int) int		// unexported method
+}
+
+// The I2 interface; all methods are exported.
+type I2 interface {
+	I0
+	F(x float) float	// exported method
+	G(x float) float	// exported method
+}
+
+// The S3 struct; all comments except for the last one must appear in the export.
+type S3 struct {
+	// lead comment for F1
+	F1	int	// line comment for F1
+	// lead comment for F2
+	F2	int	// line comment for F2
+	f3	int	// f3 is not exported
+}
+
+// This comment group should be separated
+// with a newline from the next comment
+// group.
+
+// This comment should NOT be associated with the next declaration.
+
+var x int	// x
+var ()
+
+
+// This comment SHOULD be associated with the next declaration.
+func f0() {
+	const pi = 3.14	// pi
+	var s1 struct{}	/* an empty struct */	/* foo */
+	// a struct constructor
+	// --------------------
+	var s2 struct{} = struct{}{}
+	x := pi
+}
+//
+// NO SPACE HERE
+//
+func f1() {
+	f0()
+	/* 1 */
+	// 2
+	/* 3 */
+	/* 4 */
+	f0()
+}
+
+
+func _() {
+	// this comment should be properly indented
+}
+
+
+func _(x int) int {
+	if x < 0 {	// the tab printed before this comment's // must not affect the remaining lines
+		return -x	// this statement should be properly indented
+	}
+	if x < 0 {	/* the tab printed before this comment's /* must not affect the remaining lines */
+		return -x	// this statement should be properly indented
+	}
+	return x
+}
+
+
+func typeswitch(x interface{}) {
+	switch v := x.(type) {
+	case bool, int, float:
+	case string:
+	default:
+	}
+
+	switch x.(type) {
+	}
+
+	switch v0, ok := x.(int); v := x.(type) {
+	}
+
+	switch v0, ok := x.(int); x.(type) {
+	case byte:	// this comment should be on the same line as the keyword
+		// this comment should be normally indented
+		_ = 0
+	case bool, int, float:
+		// this comment should be indented
+	case string:
+	default:
+		// this comment should be indented
+	}
+	// this comment should not be indented
+}
+
+func _() {
+	/* freestanding comment
+	   aligned		line
+	   aligned line
+	*/
+}
+
+func _() {
+	/* freestanding comment
+	   aligned		line
+	   aligned line
+	*/
+}
+
+func _() {
+	/* freestanding comment
+	   aligned		line
+	   aligned line */
+}
+
+func _() {
+	/*	freestanding comment
+		aligned		line
+		aligned line
+	*/
+}
+
+func _() {
+	/*	freestanding comment
+		aligned		line
+		aligned line
+	*/
+}
+
+func _() {
+	/*	freestanding comment
+		aligned		line
+		aligned line */
+}
+
+
+func _() {
+	/*
+	   freestanding comment
+	   aligned		line
+	   aligned line
+	*/
+}
+
+func _() {
+	/*
+	   freestanding comment
+	   aligned		line
+	   aligned line
+	*/
+}
+
+func _() {
+	/*
+	   freestanding comment
+	   aligned		line
+	   aligned line */
+}
+
+func _() {
+	/*
+		freestanding comment
+		aligned		line
+		aligned line
+	*/
+}
+
+func _() {
+	/*
+		freestanding comment
+		aligned		line
+		aligned line
+	*/
+}
+
+func _() {
+	/*
+		freestanding comment
+		aligned		line
+		aligned line */
+}
+
+func _() {
+	/* freestanding comment
+	   aligned line
+	*/
+}
+
+func _() {
+	/* freestanding comment
+	   aligned line
+	*/
+}
+
+func _() {
+	/* freestanding comment
+	   aligned line */
+}
+
+func _() {
+	/*	freestanding comment
+		aligned line
+	*/
+}
+
+func _() {
+	/*	freestanding comment
+		aligned line
+	*/
+}
+
+func _() {
+	/*	freestanding comment
+		aligned line */
+}
+
+
+func _() {
+	/*
+	   freestanding comment
+	   aligned line
+	*/
+}
+
+func _() {
+	/*
+	   freestanding comment
+	   aligned line
+	*/
+}
+
+func _() {
+	/*
+	   freestanding comment
+	   aligned line */
+}
+
+func _() {
+	/*
+		freestanding comment
+		aligned line
+	*/
+}
+
+func _() {
+	/*
+		freestanding comment
+		aligned line
+	*/
+}
+
+func _() {
+	/*
+		freestanding comment
+		aligned line */
+}
+
+/*
+ * line
+ * of
+ * stars
+ */
+
+/* another line
+ * of
+ * stars */
+
+/*	and another line
+ *	of
+ *	stars */
+
+/* a line of
+ * stars */
+
+/*	and another line of
+ *	stars */
+
+/* a line of stars
+ */
+
+/*	and another line of
+ */
+
+/* a line of stars
+ */
+
+/*	and another line of
+ */
+
+/*
+aligned in middle
+here
+        not here
+*/
+
+/*
+blank line in middle:
+
+with no leading spaces on blank line.
+*/
+
+/*
+   aligned in middle
+   here
+           not here
+*/
+
+/*
+	blank line in middle:
+
+	with no leading spaces on blank line.
+*/
+
+func _() {
+	/*
+	 * line
+	 * of
+	 * stars
+	 */
+
+	/*
+		aligned in middle
+		here
+			not here
+	*/
+
+	/*
+		blank line in middle:
+
+		with no leading spaces on blank line.
+	*/
+}
+
+
+// Some interesting interspersed comments
+func _( /* this */ x /* is */ /* an */ int) {
+}
+
+func _( /* no params */ )	{}
+
+func _() {
+	f( /* no args */ )
+}
+
+func ( /* comment1 */ T /* comment2 */ ) _()	{}
+
+func _() { /* one-line functions with comments are formatted as multi-line functions */
+}
+
+func _() {
+	_ = 0
+	/* closing curly brace should be on new line */
+}
+
+func _() {
+	_ = []int{0, 1 /* don't introduce a newline after this comment - was issue 1365 */ }
+}
+
+
+// Comments immediately adjacent to punctuation (for which the go/printer
+// may obly have estimated position information) must remain after the punctuation.
+func _() {
+	_ = T{
+		1,	// comment after comma
+		2,	/* comment after comma */
+		3,	// comment after comma
+	}
+	_ = T{
+		1,	// comment after comma
+		2,	/* comment after comma */
+		3,	// comment after comma
+	}
+	_ = T{
+		/* comment before literal */ 1,
+		2,	/* comment before comma - ok to move after comma */
+		3,	/* comment before comma - ok to move after comma */
+	}
+
+	for i = 0;	// comment after semicolon
+	i < 9;		/* comment after semicolon */
+	i++ {		// comment after opening curly brace
+	}
+
+	// TODO(gri) the last comment in this example should be aligned */
+	for i = 0;	// comment after semicolon
+	i < 9;		/* comment before semicolon - ok to move after semicolon */
+	i++ /* comment before opening curly brace */ {
+	}
+}
+
+
+// Line comments with tabs
+func _() {
+	var finput *bufio.Reader	// input file
+	var stderr *bufio.Writer
+	var ftable *bufio.Writer	// y.go file
+	var foutput *bufio.Writer	// y.output file
+
+	var oflag string	// -o [y.go]		- y.go file
+	var vflag string	// -v [y.output]	- y.output file
+	var lflag bool		// -l			- disable line directives
+}
+
+
+/* This comment is the last entry in this file. It must be printed and should be followed by a newline */
diff --git a/libgo/go/go/printer/testdata/comments.input b/libgo/go/go/printer/testdata/comments.input
new file mode 100644
index 000000000..14cd4cf7a
--- /dev/null
+++ b/libgo/go/go/printer/testdata/comments.input
@@ -0,0 +1,483 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This is a package for testing comment placement by go/printer.
+//
+package main
+
+import "fmt"  // fmt
+
+const c0 = 0  // zero
+const (
+	c1 = iota  // c1
+	c2  // c2
+)
+
+// Alignment of comments in declarations>
+const (
+	_ T = iota  // comment
+	_  // comment
+	_  // comment
+	_ = iota+10
+	_  // comments
+
+	_ = 10  // comment
+	_ T = 20  // comment
+)
+
+const (
+	_____ = iota // foo
+	_ // bar
+	_  = 0    // bal
+	_ // bat
+)
+
+const (
+	_ T = iota // comment
+	_ // comment
+	_ // comment
+	_ = iota + 10
+	_ // comment
+	_ = 10
+	_ = 20 // comment
+	_ T = 0 // comment
+)
+
+// The SZ struct; it is empty.
+type SZ struct {}
+
+// The S0 struct; no field is exported.
+type S0 struct {
+	int
+	x, y, z int  // 3 unexported fields
+}
+
+// The S1 struct; some fields are not exported.
+type S1 struct {
+	S0
+	A, B, C float  // 3 exported fields
+	D, b, c int  // 2 unexported fields
+}
+
+// The S2 struct; all fields are exported.
+type S2 struct {
+	S1
+	A, B, C float  // 3 exported fields
+}
+
+// The IZ interface; it is empty.
+type SZ interface {}
+
+// The I0 interface; no method is exported.
+type I0 interface {
+	f(x int) int  // unexported method
+}
+
+// The I1 interface; some methods are not exported.
+type I1 interface {
+	I0
+	F(x float) float  // exported methods
+	g(x int) int  // unexported method
+}
+
+// The I2 interface; all methods are exported.
+type I2 interface {
+	I0
+	F(x float) float  // exported method
+	G(x float) float  // exported method
+}
+
+// The S3 struct; all comments except for the last one must appear in the export.
+type S3 struct {
+	// lead comment for F1
+	F1 int // line comment for F1
+	// lead comment for F2
+	F2 int // line comment for F2
+	f3 int // f3 is not exported
+}
+
+// This comment group should be separated
+// with a newline from the next comment
+// group.
+
+// This comment should NOT be associated with the next declaration.
+
+var x int  // x
+var ()
+
+
+// This comment SHOULD be associated with the next declaration.
+func f0() {
+	const pi = 3.14  // pi
+	var s1 struct {}  /* an empty struct */ /* foo */
+	// a struct constructor
+	// --------------------
+	var s2 struct {} = struct {}{}
+	x := pi
+}
+//
+// NO SPACE HERE
+//
+func f1() {
+	f0()
+	/* 1 */
+	// 2
+	/* 3 */
+	/* 4 */
+	f0()
+}
+
+
+func _() {
+	// this comment should be properly indented
+}
+
+
+func _(x int) int {
+	if x < 0 {  // the tab printed before this comment's // must not affect the remaining lines
+		return -x  // this statement should be properly indented
+	}
+	if x < 0 {  /* the tab printed before this comment's /* must not affect the remaining lines */
+		return -x  // this statement should be properly indented
+	}
+	return x
+}
+
+
+func typeswitch(x interface{}) {
+	switch v := x.(type) {
+	case bool, int, float:
+	case string:
+	default:
+	}
+
+	switch x.(type) {
+	}
+
+	switch v0, ok := x.(int); v := x.(type) {
+	}
+
+	switch v0, ok := x.(int); x.(type) {
+	case byte:  // this comment should be on the same line as the keyword
+		// this comment should be normally indented
+		_ = 0
+	case bool, int, float:
+		// this comment should be indented
+	case string:
+	default:
+		// this comment should be indented
+	}
+	// this comment should not be indented
+}
+
+func _() {
+	/* freestanding comment
+	   aligned		line
+	   aligned line
+	*/
+}
+
+func _() {
+	/* freestanding comment
+	   aligned		line
+	   aligned line
+	   */
+}
+
+func _() {
+	/* freestanding comment
+	   aligned		line
+	   aligned line */
+}
+
+func _() {
+	/*	freestanding comment
+		aligned		line
+		aligned line
+	*/
+}
+
+func _() {
+	/*	freestanding comment
+		aligned		line
+		aligned line
+		*/
+}
+
+func _() {
+	/*	freestanding comment
+		aligned		line
+		aligned line */
+}
+
+
+func _() {
+	/*
+	   freestanding comment
+	   aligned		line
+	   aligned line
+	*/
+}
+
+func _() {
+	/*
+	   freestanding comment
+	   aligned		line
+	   aligned line
+	   */
+}
+
+func _() {
+	/*
+	   freestanding comment
+	   aligned		line
+	   aligned line */
+}
+
+func _() {
+	/*
+		freestanding comment
+		aligned		line
+		aligned line
+	*/
+}
+
+func _() {
+	/*
+		freestanding comment
+		aligned		line
+		aligned line
+		*/
+}
+
+func _() {
+	/*
+		freestanding comment
+		aligned		line
+		aligned line */
+}
+
+func _() {
+	/* freestanding comment
+	   aligned line
+	*/
+}
+
+func _() {
+	/* freestanding comment
+	   aligned line
+	   */
+}
+
+func _() {
+	/* freestanding comment
+	   aligned line */
+}
+
+func _() {
+	/*	freestanding comment
+		aligned line
+	*/
+}
+
+func _() {
+	/*	freestanding comment
+		aligned line
+		*/
+}
+
+func _() {
+	/*	freestanding comment
+		aligned line */
+}
+
+
+func _() {
+	/*
+	   freestanding comment
+	   aligned line
+	*/
+}
+
+func _() {
+	/*
+	   freestanding comment
+	   aligned line
+	   */
+}
+
+func _() {
+	/*
+	   freestanding comment
+	   aligned line */
+}
+
+func _() {
+	/*
+		freestanding comment
+		aligned line
+	*/
+}
+
+func _() {
+	/*
+		freestanding comment
+		aligned line
+		*/
+}
+
+func _() {
+	/*
+		freestanding comment
+		aligned line */
+}
+
+/*
+ * line
+ * of
+ * stars
+ */
+
+/* another line
+ * of
+ * stars */
+
+/*	and another line
+ *	of
+ *	stars */
+
+/* a line of
+ * stars */
+
+/*	and another line of
+ *	stars */
+
+/* a line of stars
+*/
+
+/*	and another line of
+*/
+
+/* a line of stars
+ */
+
+/*	and another line of
+ */
+
+/*
+aligned in middle
+here
+        not here
+*/
+
+/*
+blank line in middle:
+
+with no leading spaces on blank line.
+*/
+
+/*
+   aligned in middle
+   here
+           not here
+*/
+
+/*
+	blank line in middle:
+
+	with no leading spaces on blank line.
+*/
+
+func _() {
+	/*
+	 * line
+	 * of
+	 * stars
+	 */
+
+	/*
+	aligned in middle
+	here
+		not here
+	*/
+
+	/*
+	blank line in middle:
+
+	with no leading spaces on blank line.
+*/
+}
+
+
+// Some interesting interspersed comments
+func _(/* this */x/* is *//* an */ int) {
+}
+
+func _(/* no params */) {}
+
+func _() {
+	f(/* no args */)
+}
+
+func (/* comment1 */ T /* comment2 */) _() {}
+
+func _() { /* one-line functions with comments are formatted as multi-line functions */ }
+
+func _() {
+	_ = 0
+	/* closing curly brace should be on new line */ }
+
+func _() {
+	_ = []int{0, 1 /* don't introduce a newline after this comment - was issue 1365 */}
+}
+
+
+// Comments immediately adjacent to punctuation (for which the go/printer
+// may obly have estimated position information) must remain after the punctuation.
+func _() {
+	_ = T{
+		1,    // comment after comma
+		2,    /* comment after comma */
+		3  ,  // comment after comma
+	}
+	_ = T{
+		1  ,// comment after comma
+		2  ,/* comment after comma */
+		3,// comment after comma
+	}
+	_ = T{
+		/* comment before literal */1,
+		2/* comment before comma - ok to move after comma */,
+		3  /* comment before comma - ok to move after comma */  ,
+	}
+
+	for
+		i=0;// comment after semicolon
+		i<9;/* comment after semicolon */
+		i++{// comment after opening curly brace
+	}
+
+	// TODO(gri) the last comment in this example should be aligned */
+	for
+		i=0;// comment after semicolon
+		i<9/* comment before semicolon - ok to move after semicolon */;
+		i++ /* comment before opening curly brace */ {
+	}
+}
+
+
+// Line comments with tabs
+func _() {
+var	finput		*bufio.Reader			// input file
+var	stderr		*bufio.Writer
+var	ftable		*bufio.Writer			// y.go file
+var	foutput		*bufio.Writer			// y.output file
+
+var	oflag		string				// -o [y.go]		- y.go file
+var	vflag		string				// -v [y.output]	- y.output file
+var	lflag		bool				// -l			- disable line directives
+}
+
+
+/* This comment is the last entry in this file. It must be printed and should be followed by a newline */
diff --git a/libgo/go/go/printer/testdata/comments.x b/libgo/go/go/printer/testdata/comments.x
new file mode 100644
index 000000000..4d7a928ae
--- /dev/null
+++ b/libgo/go/go/printer/testdata/comments.x
@@ -0,0 +1,57 @@
+// This is a package for testing comment placement by go/printer.
+//
+package main
+
+
+// The SZ struct; it is empty.
+type SZ struct{}
+
+// The S0 struct; no field is exported.
+type S0 struct {
+	// contains unexported fields
+}
+
+// The S1 struct; some fields are not exported.
+type S1 struct {
+	S0
+	A, B, C	float	// 3 exported fields
+	D	int	// 2 unexported fields
+	// contains unexported fields
+}
+
+// The S2 struct; all fields are exported.
+type S2 struct {
+	S1
+	A, B, C	float	// 3 exported fields
+}
+
+// The IZ interface; it is empty.
+type SZ interface{}
+
+// The I0 interface; no method is exported.
+type I0 interface {
+	// contains unexported methods
+}
+
+// The I1 interface; some methods are not exported.
+type I1 interface {
+	I0
+	F(x float) float	// exported methods
+	// contains unexported methods
+}
+
+// The I2 interface; all methods are exported.
+type I2 interface {
+	I0
+	F(x float) float	// exported method
+	G(x float) float	// exported method
+}
+
+// The S3 struct; all comments except for the last one must appear in the export.
+type S3 struct {
+	// lead comment for F1
+	F1	int	// line comment for F1
+	// lead comment for F2
+	F2	int	// line comment for F2
+	// contains unexported fields
+}
diff --git a/libgo/go/go/printer/testdata/declarations.golden b/libgo/go/go/printer/testdata/declarations.golden
new file mode 100644
index 000000000..1c091b929
--- /dev/null
+++ b/libgo/go/go/printer/testdata/declarations.golden
@@ -0,0 +1,715 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package imports
+
+import "io"
+
+import (
+	_	"io"
+)
+
+import _	"io"
+
+import (
+	"io"
+	"io"
+	"io"
+)
+
+import (
+	"io"
+	aLongRename	"io"
+
+	b	"io"
+)
+
+import (
+	"unrenamed"
+	renamed	"renameMe"
+	.	"io"
+	_	"io"
+	"io"
+	.	"os"
+)
+
+// no newlines between consecutive single imports, but
+// respect extra line breaks in the source (at most one empty line)
+import _	"io"
+import _	"io"
+import _	"io"
+
+import _	"os"
+import _	"os"
+import _	"os"
+
+
+import _	"fmt"
+import _	"fmt"
+import _	"fmt"
+
+import "foo"	// a comment
+import "bar"	// a comment
+
+import (
+	_	"foo"
+	// a comment
+	"bar"
+	"foo"	// a comment
+	"bar"	// a comment
+)
+
+// comments + renames
+import (
+	"unrenamed"	// a comment
+	renamed		"renameMe"
+	.		"io"		/* a comment */
+	_		"io/ioutil"	// a comment
+	"io"		// testing alignment
+	.		"os"
+	// a comment
+)
+
+// a case that caused problems in the past (comment placement)
+import (
+	.	"fmt"
+	"io"
+	"malloc"	// for the malloc count test only
+	"math"
+	"strings"
+	"testing"
+)
+
+
+// at least one empty line between declarations of different kind
+import _	"io"
+
+var _ int
+
+
+// printing of constant literals
+const (
+	_	= "foobar"
+	_	= "a۰۱۸"
+	_	= "foo६४"
+	_	= "bar９８７６"
+	_	= 0
+	_	= 1
+	_	= 123456789012345678890
+	_	= 01234567
+	_	= 0xcafebabe
+	_	= 0.
+	_	= .0
+	_	= 3.14159265
+	_	= 1e0
+	_	= 1e+100
+	_	= 1e-100
+	_	= 2.71828e-1000
+	_	= 0i
+	_	= 1i
+	_	= 012345678901234567889i
+	_	= 123456789012345678890i
+	_	= 0.i
+	_	= .0i
+	_	= 3.14159265i
+	_	= 1e0i
+	_	= 1e+100i
+	_	= 1e-100i
+	_	= 2.71828e-1000i
+	_	= 'a'
+	_	= '\000'
+	_	= '\xFF'
+	_	= '\uff16'
+	_	= '\U0000ff16'
+	_	= `foobar`
+	_	= `foo
+---
+---
+bar`
+)
+
+
+func _() {
+	// the following decls need a semicolon at the end
+	type _ int
+	type _ *int
+	type _ []int
+	type _ map[string]int
+	type _ chan int
+	type _ func() int
+
+	var _ int
+	var _ *int
+	var _ []int
+	var _ map[string]int
+	var _ chan int
+	var _ func() int
+
+	// the following decls don't need a semicolon at the end
+	type _ struct{}
+	type _ *struct{}
+	type _ []struct{}
+	type _ map[string]struct{}
+	type _ chan struct{}
+	type _ func() struct{}
+
+	type _ interface{}
+	type _ *interface{}
+	type _ []interface{}
+	type _ map[string]interface{}
+	type _ chan interface{}
+	type _ func() interface{}
+
+	var _ struct{}
+	var _ *struct{}
+	var _ []struct{}
+	var _ map[string]struct{}
+	var _ chan struct{}
+	var _ func() struct{}
+
+	var _ interface{}
+	var _ *interface{}
+	var _ []interface{}
+	var _ map[string]interface{}
+	var _ chan interface{}
+	var _ func() interface{}
+}
+
+
+// don't lose blank lines in grouped declarations
+const (
+	_	int	= 0
+	_	float	= 1
+
+	_	string	= "foo"
+
+	_	= iota
+	_
+
+	// a comment
+	_
+
+	_
+)
+
+
+type (
+	_	int
+	_	struct{}
+
+	_	interface{}
+
+	// a comment
+	_	map[string]int
+)
+
+
+var (
+	_	int	= 0
+	_	float	= 1
+
+	_	string	= "foo"
+
+	_	bool
+
+	// a comment
+	_	bool
+)
+
+
+// don't lose blank lines in this struct
+type _ struct {
+	String	struct {
+		Str, Len int
+	}
+	Slice	struct {
+		Array, Len, Cap int
+	}
+	Eface	struct {
+		Typ, Ptr int
+	}
+
+	UncommonType	struct {
+		Name, PkgPath int
+	}
+	CommonType	struct {
+		Size, Hash, Alg, Align, FieldAlign, String, UncommonType int
+	}
+	Type	struct {
+		Typ, Ptr int
+	}
+	StructField	struct {
+		Name, PkgPath, Typ, Tag, Offset int
+	}
+	StructType	struct {
+		Fields int
+	}
+	PtrType	struct {
+		Elem int
+	}
+	SliceType	struct {
+		Elem int
+	}
+	ArrayType	struct {
+		Elem, Len int
+	}
+
+	Stktop	struct {
+		Stackguard, Stackbase, Gobuf int
+	}
+	Gobuf	struct {
+		Sp, Pc, G int
+	}
+	G	struct {
+		Stackbase, Sched, Status, Alllink int
+	}
+}
+
+
+// no tabs for single or ungrouped decls
+func _() {
+	const xxxxxx = 0
+	type x int
+	var xxx int
+	var yyyy float = 3.14
+	var zzzzz = "bar"
+
+	const (
+		xxxxxx = 0
+	)
+	type (
+		x int
+	)
+	var (
+		xxx int
+	)
+	var (
+		yyyy float = 3.14
+	)
+	var (
+		zzzzz = "bar"
+	)
+}
+
+// tabs for multiple or grouped decls
+func _() {
+	// no entry has a type
+	const (
+		zzzzzz	= 1
+		z	= 2
+		zzz	= 3
+	)
+	// some entries have a type
+	const (
+		xxxxxx		= 1
+		x		= 2
+		xxx		= 3
+		yyyyyyyy	float	= iota
+		yyyy		= "bar"
+		yyy
+		yy	= 2
+	)
+}
+
+func _() {
+	// no entry has a type
+	var (
+		zzzzzz	= 1
+		z	= 2
+		zzz	= 3
+	)
+	// no entry has a value
+	var (
+		_	int
+		_	float
+		_	string
+
+		_	int	// comment
+		_	float	// comment
+		_	string	// comment
+	)
+	// some entries have a type
+	var (
+		xxxxxx		int
+		x		float
+		xxx		string
+		yyyyyyyy	int	= 1234
+		y		float	= 3.14
+		yyyy		= "bar"
+		yyy		string	= "foo"
+	)
+	// mixed entries - all comments should be aligned
+	var (
+		a, b, c			int
+		x			= 10
+		d			int			// comment
+		y			= 20			// comment
+		f, ff, fff, ffff	int	= 0, 1, 2, 3	// comment
+	)
+	// respect original line breaks
+	var _ = []T{
+		T{0x20, "Telugu"},
+	}
+	var _ = []T{
+		// respect original line breaks
+		T{0x20, "Telugu"},
+	}
+}
+
+func _() {
+	type (
+		xxxxxx	int
+		x	float
+		xxx	string
+		xxxxx	[]x
+		xx	struct{}
+		xxxxxxx	struct {
+			_, _	int
+			_	float
+		}
+		xxxx	chan<- string
+	)
+}
+
+
+// formatting of structs
+type _ struct{}
+
+type _ struct { /* this comment should be visible */
+}
+
+type _ struct {
+	// this comment should be visible and properly indented
+}
+
+type _ struct {	// this comment must not change indentation
+	f			int
+	f, ff, fff, ffff	int
+}
+
+type _ struct {
+	string
+}
+
+type _ struct {
+	string	// comment
+}
+
+type _ struct {
+	string "tag"
+}
+
+type _ struct {
+	string "tag"	// comment
+}
+
+type _ struct {
+	f int
+}
+
+type _ struct {
+	f int	// comment
+}
+
+type _ struct {
+	f int "tag"
+}
+
+type _ struct {
+	f int "tag"	// comment
+}
+
+type _ struct {
+	bool
+	a, b, c			int
+	int			"tag"
+	ES				// comment
+	float			"tag"	// comment
+	f			int	// comment
+	f, ff, fff, ffff	int	// comment
+	g			float	"tag"
+	h			float	"tag"	// comment
+}
+
+type _ struct {
+	a, b,
+	c, d	int	// this line should be indented
+	u, v, w, x	float	// this line should be indented
+	p, q,
+	r, s	float	// this line should be indented
+}
+
+
+// difficult cases
+type _ struct {
+	bool		// comment
+	text	[]byte	// comment
+}
+
+
+// formatting of interfaces
+type EI interface{}
+
+type _ interface {
+	EI
+}
+
+type _ interface {
+	f()
+	fffff()
+}
+
+type _ interface {
+	EI
+	f()
+	fffffg()
+}
+
+type _ interface {	// this comment must not change indentation
+	EI				// here's a comment
+	f()				// no blank between identifier and ()
+	fffff()				// no blank between identifier and ()
+	gggggggggggg(x, y, z int)	// hurray
+}
+
+
+// formatting of variable declarations
+func _() {
+	type day struct {
+		n		int
+		short, long	string
+	}
+	var (
+		Sunday		= day{0, "SUN", "Sunday"}
+		Monday		= day{1, "MON", "Monday"}
+		Tuesday		= day{2, "TUE", "Tuesday"}
+		Wednesday	= day{3, "WED", "Wednesday"}
+		Thursday	= day{4, "THU", "Thursday"}
+		Friday		= day{5, "FRI", "Friday"}
+		Saturday	= day{6, "SAT", "Saturday"}
+	)
+}
+
+
+// formatting of multi-line variable declarations
+var a1, b1, c1 int	// all on one line
+
+var a2, b2,
+	c2 int	// this line should be indented
+
+var (
+	a3, b3,
+	c3, d3	int	// this line should be indented
+	a4, b4, c4	int	// this line should be indented
+)
+
+
+func _() {
+	var privateKey2 = &Block{Type:	"RSA PRIVATE KEY",
+		Headers:	map[string]string{},
+		Bytes: []uint8{0x30, 0x82, 0x1, 0x3a, 0x2, 0x1, 0x0, 0x2,
+			0x41, 0x0, 0xb2, 0x99, 0xf, 0x49, 0xc4, 0x7d, 0xfa, 0x8c,
+			0xd4, 0x0, 0xae, 0x6a, 0x4d, 0x1b, 0x8a, 0x3b, 0x6a, 0x13,
+			0x64, 0x2b, 0x23, 0xf2, 0x8b, 0x0, 0x3b, 0xfb, 0x97, 0x79,
+		},
+	}
+}
+
+
+func _() {
+	var Universe = Scope{
+		Names: map[string]*Ident{
+			// basic types
+			"bool":		nil,
+			"byte":		nil,
+			"int8":		nil,
+			"int16":	nil,
+			"int32":	nil,
+			"int64":	nil,
+			"uint8":	nil,
+			"uint16":	nil,
+			"uint32":	nil,
+			"uint64":	nil,
+			"float32":	nil,
+			"float64":	nil,
+			"string":	nil,
+
+			// convenience types
+			"int":		nil,
+			"uint":		nil,
+			"uintptr":	nil,
+			"float":	nil,
+
+			// constants
+			"false":	nil,
+			"true":		nil,
+			"iota":		nil,
+			"nil":		nil,
+
+			// functions
+			"cap":		nil,
+			"len":		nil,
+			"new":		nil,
+			"make":		nil,
+			"panic":	nil,
+			"panicln":	nil,
+			"print":	nil,
+			"println":	nil,
+		},
+	}
+}
+
+
+// alignment of map composite entries
+var _ = map[int]int{
+	// small key sizes: always align even if size ratios are large
+	a:			a,
+	abcdefghabcdefgh:	a,
+	ab:			a,
+	abc:			a,
+	abcdefgabcdefg:		a,
+	abcd:			a,
+	abcde:			a,
+	abcdef:			a,
+
+	// mixed key sizes: align when key sizes change within accepted ratio
+	abcdefgh:		a,
+	abcdefghabcdefg:	a,
+	abcdefghij:		a,
+	abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij:	a,	// outlier - do not align with previous line
+	abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij:		a,	// align with previous line
+
+	ab:	a,	// do not align with previous line
+	abcde:	a,	// align with previous line
+}
+
+
+func _() {
+	var _ = T{
+		a,	// must introduce trailing comma
+	}
+}
+
+
+// formatting of function results
+func _() func()				{}
+func _() func(int)			{ return nil }
+func _() func(int) int			{ return nil }
+func _() func(int) func(int) func()	{ return nil }
+
+
+// formatting of consecutive single-line functions
+func _()	{}
+func _()	{}
+func _()	{}
+
+func _()	{}	// an empty line before this function
+func _()	{}
+func _()	{}
+
+func _()		{ f(1, 2, 3) }
+func _(x int) int	{ y := x; return y + 1 }
+func _() int		{ type T struct{}; var x T; return x }
+
+// these must remain multi-line since they are multi-line in the source
+func _() {
+	f(1, 2, 3)
+}
+func _(x int) int {
+	y := x
+	return y + 1
+}
+func _() int {
+	type T struct{}
+	var x T
+	return x
+}
+
+
+// making function declarations safe for new semicolon rules
+func _() { /* multi-line func because of comment */
+}
+
+func _() {
+	/* multi-line func because block is on multiple lines */
+}
+
+
+// ellipsis parameters
+func _(...int)
+func _(...*int)
+func _(...[]int)
+func _(...struct{})
+func _(bool, ...interface{})
+func _(bool, ...func())
+func _(bool, ...func(...int))
+func _(bool, ...map[string]int)
+func _(bool, ...chan int)
+
+func _(b bool, x ...int)
+func _(b bool, x ...*int)
+func _(b bool, x ...[]int)
+func _(b bool, x ...struct{})
+func _(x ...interface{})
+func _(x ...func())
+func _(x ...func(...int))
+func _(x ...map[string]int)
+func _(x ...chan int)
+
+
+// these parameter lists must remain multi-line since they are multi-line in the source
+func _(bool,
+int) {
+}
+func _(x bool,
+y int) {
+}
+func _(x,
+y bool) {
+}
+func _(bool,	// comment
+int) {
+}
+func _(x bool,	// comment
+y int) {
+}
+func _(x,	// comment
+y bool) {
+}
+func _(bool,	// comment
+// comment
+int) {
+}
+func _(x bool,	// comment
+// comment
+y int) {
+}
+func _(x,	// comment
+// comment
+y bool) {
+}
+func _(bool,
+// comment
+int) {
+}
+func _(x bool,
+// comment
+y int) {
+}
+func _(x,
+// comment
+y bool) {
+}
+func _(x,	// comment
+y,	// comment
+z bool) {
+}
+func _(x,	// comment
+y,	// comment
+z bool) {
+}
+func _(x int,	// comment
+y float,	// comment
+z bool) {
+}
diff --git a/libgo/go/go/printer/testdata/declarations.input b/libgo/go/go/printer/testdata/declarations.input
new file mode 100644
index 000000000..c826462f9
--- /dev/null
+++ b/libgo/go/go/printer/testdata/declarations.input
@@ -0,0 +1,703 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package imports
+
+import "io"
+
+import (
+	_ "io"
+)
+
+import _ "io"
+
+import (
+	"io"
+	"io"
+	"io"
+)
+
+import (
+	"io"
+	aLongRename "io"
+
+	b "io"
+)
+
+import (
+       "unrenamed"
+       renamed "renameMe"
+       . "io"
+       _ "io"
+       "io"
+       . "os"
+)
+
+// no newlines between consecutive single imports, but
+// respect extra line breaks in the source (at most one empty line)
+import _ "io"
+import _ "io"
+import _ "io"
+
+import _ "os"
+import _ "os"
+import _ "os"
+
+
+import _ "fmt"
+import _ "fmt"
+import _ "fmt"
+
+import "foo"  // a comment
+import "bar"  // a comment
+
+import (
+	_ "foo"
+	// a comment
+	"bar"
+	"foo"  // a comment
+	"bar"  // a comment
+)
+
+// comments + renames
+import (
+       "unrenamed" // a comment
+       renamed "renameMe"
+       . "io" /* a comment */
+       _ "io/ioutil" // a comment
+       "io" // testing alignment
+       . "os"
+       // a comment
+)
+
+// a case that caused problems in the past (comment placement)
+import (
+	. "fmt"
+	"io"
+	"malloc"	// for the malloc count test only
+	"math"
+	"strings"
+	"testing"
+)
+
+
+// at least one empty line between declarations of different kind
+import _ "io"
+var _ int
+
+
+// printing of constant literals
+const (
+	_ = "foobar"
+	_ = "a۰۱۸"
+	_ = "foo६४"
+	_ = "bar９８７６"
+	_ = 0
+	_ = 1
+	_ = 123456789012345678890
+	_ = 01234567
+	_ = 0xcafebabe
+	_ = 0.
+	_ = .0
+	_ = 3.14159265
+	_ = 1e0
+	_ = 1e+100
+	_ = 1e-100
+	_ = 2.71828e-1000
+	_ = 0i
+	_ = 1i
+	_ = 012345678901234567889i
+	_ = 123456789012345678890i
+	_ = 0.i
+	_ = .0i
+	_ = 3.14159265i
+	_ = 1e0i
+	_ = 1e+100i
+	_ = 1e-100i
+	_ = 2.71828e-1000i
+	_ = 'a'
+	_ = '\000'
+	_ = '\xFF'
+	_ = '\uff16'
+	_ = '\U0000ff16'
+	_ = `foobar`
+	_ = `foo
+---
+---
+bar`
+)
+
+
+func _() {
+	// the following decls need a semicolon at the end
+	type _ int
+	type _ *int
+	type _ []int
+	type _ map[string]int
+	type _ chan int
+	type _ func() int
+
+	var _ int
+	var _ *int
+	var _ []int
+	var _ map[string]int
+	var _ chan int
+	var _ func() int
+
+	// the following decls don't need a semicolon at the end
+	type _ struct{}
+	type _ *struct{}
+	type _ []struct{}
+	type _ map[string]struct{}
+	type _ chan struct{}
+	type _ func() struct{}
+
+	type _ interface{}
+	type _ *interface{}
+	type _ []interface{}
+	type _ map[string]interface{}
+	type _ chan interface{}
+	type _ func() interface{}
+
+	var _ struct{}
+	var _ *struct{}
+	var _ []struct{}
+	var _ map[string]struct{}
+	var _ chan struct{}
+	var _ func() struct{}
+
+	var _ interface{}
+	var _ *interface{}
+	var _ []interface{}
+	var _ map[string]interface{}
+	var _ chan interface{}
+	var _ func() interface{}
+}
+
+
+// don't lose blank lines in grouped declarations
+const (
+	_ int = 0
+	_ float = 1
+
+	_ string = "foo"
+
+	_ = iota
+	_
+	
+	// a comment
+	_
+
+	_
+)
+
+
+type (
+	_ int
+	_ struct {}
+	
+	_ interface{}
+	
+	// a comment
+	_ map[string]int
+)
+
+
+var (
+	_ int = 0
+	_ float = 1
+
+	_ string = "foo"
+
+	_ bool
+	
+	// a comment
+	_ bool
+)
+
+
+// don't lose blank lines in this struct
+type _ struct {
+	String struct {
+		Str, Len int
+	}
+	Slice struct {
+		Array, Len, Cap int
+	}
+	Eface struct {
+		Typ, Ptr int
+	}
+
+	UncommonType struct {
+		Name, PkgPath int
+	}
+	CommonType struct {
+		Size, Hash, Alg, Align, FieldAlign, String, UncommonType int
+	}
+	Type struct {
+		Typ, Ptr int
+	}
+	StructField struct {
+		Name, PkgPath, Typ, Tag, Offset int
+	}
+	StructType struct {
+		Fields int
+	}
+	PtrType struct {
+		Elem int
+	}
+	SliceType struct {
+		Elem int
+	}
+	ArrayType struct {
+		Elem, Len int
+	}
+
+	Stktop struct {
+		Stackguard, Stackbase, Gobuf int
+	}
+	Gobuf struct {
+		Sp, Pc, G int
+	}
+	G struct {
+		Stackbase, Sched, Status, Alllink int
+	}
+}
+
+
+// no tabs for single or ungrouped decls
+func _() {
+	const xxxxxx = 0
+	type x int
+	var xxx int
+	var yyyy float = 3.14
+	var zzzzz = "bar"
+
+	const (
+		xxxxxx = 0
+	)
+	type (
+		x int
+	)
+	var (
+		xxx int
+	)
+	var (
+		yyyy float = 3.14
+	)
+	var (
+		zzzzz = "bar"
+	)
+}
+
+// tabs for multiple or grouped decls
+func _() {
+	// no entry has a type
+	const (
+		zzzzzz = 1
+		z = 2
+		zzz = 3
+	)
+	// some entries have a type
+	const (
+		xxxxxx = 1
+		x = 2
+		xxx = 3
+		yyyyyyyy float = iota
+		yyyy = "bar"
+		yyy
+		yy = 2
+	)
+}
+
+func _() {
+	// no entry has a type
+	var (
+		zzzzzz = 1
+		z = 2
+		zzz = 3
+	)
+	// no entry has a value
+	var (
+		_ int
+		_ float
+		_ string
+
+		_ int  // comment
+		_ float  // comment
+		_ string  // comment
+	)
+	// some entries have a type
+	var (
+		xxxxxx int
+		x float
+		xxx string
+		yyyyyyyy int = 1234
+		y float = 3.14
+		yyyy = "bar"
+		yyy string = "foo"
+	)
+	// mixed entries - all comments should be aligned
+	var (
+		a, b, c int
+		x = 10
+		d int  // comment
+		y = 20  // comment
+		f, ff, fff, ffff int = 0, 1, 2, 3  // comment
+	)
+	// respect original line breaks
+	var _ = []T {
+		T{0x20,	"Telugu"},
+	}
+	var _ = []T {
+		// respect original line breaks
+		T{0x20,	"Telugu"},
+	}
+}
+
+func _() {
+	type (
+		xxxxxx int
+		x float
+		xxx string
+		xxxxx []x
+		xx struct{}
+		xxxxxxx struct {
+			_, _ int
+			_ float
+		}
+		xxxx chan<- string
+	)
+}
+
+
+// formatting of structs
+type _ struct{}
+
+type _ struct{ /* this comment should be visible */ }
+
+type _ struct{
+	// this comment should be visible and properly indented
+}
+
+type _ struct {  // this comment must not change indentation
+	f int
+	f, ff, fff, ffff int
+}
+
+type _ struct {
+	string
+}
+
+type _ struct {
+	string  // comment
+}
+
+type _ struct {
+	string "tag"
+}
+
+type _ struct {
+	string "tag"  // comment
+}
+
+type _ struct {
+	f int
+}
+
+type _ struct {
+	f int  // comment
+}
+
+type _ struct {
+	f int "tag"
+}
+
+type _ struct {
+	f int "tag"  // comment
+}
+
+type _ struct {
+	bool
+	a, b, c int
+	int "tag"
+	ES // comment
+	float "tag"  // comment
+	f int  // comment
+	f, ff, fff, ffff int  // comment
+	g float "tag"
+	h float "tag"  // comment
+}
+
+type _ struct { a, b,
+c, d int  // this line should be indented
+u, v, w, x float // this line should be indented
+p, q,
+r, s float // this line should be indented
+}
+
+
+// difficult cases
+type _ struct {
+	bool  // comment
+	text []byte  // comment
+}
+
+
+// formatting of interfaces
+type EI interface{}
+
+type _ interface {
+	EI
+}
+
+type _ interface {
+	f()
+	fffff()
+}
+
+type _ interface {
+	EI
+	f()
+	fffffg()
+}
+
+type _ interface {  // this comment must not change indentation
+	EI  // here's a comment
+	f()  // no blank between identifier and ()
+	fffff()  // no blank between identifier and ()
+	gggggggggggg(x, y, z int) ()  // hurray
+}
+
+
+// formatting of variable declarations
+func _() {
+	type day struct { n int; short, long string }
+	var (
+		Sunday = day{ 0, "SUN", "Sunday" }
+		Monday = day{ 1, "MON", "Monday" }
+		Tuesday = day{ 2, "TUE", "Tuesday" }
+		Wednesday = day{ 3, "WED", "Wednesday" }
+		Thursday = day{ 4, "THU", "Thursday" }
+		Friday = day{ 5, "FRI", "Friday" }
+		Saturday = day{ 6, "SAT", "Saturday" }
+	)
+}
+
+
+// formatting of multi-line variable declarations
+var a1, b1, c1 int  // all on one line
+
+var a2, b2,
+c2 int  // this line should be indented
+
+var (a3, b3,
+c3, d3 int  // this line should be indented
+a4, b4, c4 int  // this line should be indented
+)
+
+
+func _() {
+	var privateKey2 = &Block{Type: "RSA PRIVATE KEY",
+					Headers: map[string]string{},
+					Bytes: []uint8{0x30, 0x82, 0x1, 0x3a, 0x2, 0x1, 0x0, 0x2,
+			0x41, 0x0, 0xb2, 0x99, 0xf, 0x49, 0xc4, 0x7d, 0xfa, 0x8c,
+			0xd4, 0x0, 0xae, 0x6a, 0x4d, 0x1b, 0x8a, 0x3b, 0x6a, 0x13,
+			0x64, 0x2b, 0x23, 0xf2, 0x8b, 0x0, 0x3b, 0xfb, 0x97, 0x79,
+		},
+	}
+}
+
+
+func _() {
+	var Universe = Scope {
+		Names: map[string]*Ident {
+			// basic types
+			"bool": nil,
+			"byte": nil,
+			"int8": nil,
+			"int16": nil,
+			"int32": nil,
+			"int64": nil,
+			"uint8": nil,
+			"uint16": nil,
+			"uint32": nil,
+			"uint64": nil,
+			"float32": nil,
+			"float64": nil,
+			"string": nil,
+
+			// convenience types
+			"int": nil,
+			"uint": nil,
+			"uintptr": nil,
+			"float": nil,
+
+			// constants
+			"false": nil,
+			"true": nil,
+			"iota": nil,
+			"nil": nil,
+
+			// functions
+			"cap": nil,
+			"len": nil,
+			"new": nil,
+			"make": nil,
+			"panic": nil,
+			"panicln": nil,
+			"print": nil,
+			"println": nil,
+		},
+	}
+}
+
+
+// alignment of map composite entries
+var _ = map[int]int{
+	// small key sizes: always align even if size ratios are large
+	a: a,
+	abcdefghabcdefgh: a,
+	ab: a,
+	abc: a,
+	abcdefgabcdefg: a,
+	abcd: a,
+	abcde: a,
+	abcdef: a,
+
+	// mixed key sizes: align when key sizes change within accepted ratio
+	abcdefgh: a,
+	abcdefghabcdefg: a,
+	abcdefghij: a,
+	abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij: a, // outlier - do not align with previous line
+	abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij: a, // align with previous line
+
+	ab: a, // do not align with previous line
+	abcde: a, // align with previous line
+}
+
+
+func _() {
+	var _ = T{
+		a,	// must introduce trailing comma
+	}
+}
+
+
+// formatting of function results
+func _() func() {}
+func _() func(int) { return nil }
+func _() func(int) int { return nil }
+func _() func(int) func(int) func() { return nil }
+
+
+// formatting of consecutive single-line functions
+func _() {}
+func _() {}
+func _() {}
+
+func _() {}  // an empty line before this function
+func _() {}
+func _() {}
+
+func _() { f(1, 2, 3) }
+func _(x int) int { y := x; return y+1 }
+func _() int { type T struct{}; var x T; return x }
+
+// these must remain multi-line since they are multi-line in the source
+func _() {
+	f(1, 2, 3)
+}
+func _(x int) int {
+	y := x; return y+1
+}
+func _() int {
+	type T struct{}; var x T; return x
+}
+
+
+// making function declarations safe for new semicolon rules
+func _() { /* multi-line func because of comment */ }
+
+func _() {
+/* multi-line func because block is on multiple lines */ }
+
+
+// ellipsis parameters
+func _(...int)
+func _(...*int)
+func _(...[]int)
+func _(...struct{})
+func _(bool, ...interface{})
+func _(bool, ...func())
+func _(bool, ...func(...int))
+func _(bool, ...map[string]int)
+func _(bool, ...chan int)
+
+func _(b bool, x ...int)
+func _(b bool, x ...*int)
+func _(b bool, x ...[]int)
+func _(b bool, x ...struct{})
+func _(x ...interface{})
+func _(x ...func())
+func _(x ...func(...int))
+func _(x ...map[string]int)
+func _(x ...chan int)
+
+
+// these parameter lists must remain multi-line since they are multi-line in the source
+func _(bool,
+int) {
+}
+func _(x bool,
+y int) {
+}
+func _(x,
+y bool) {
+}
+func _(bool, // comment
+int) {
+}
+func _(x bool, // comment
+y int) {
+}
+func _(x, // comment
+y bool) {
+}
+func _(bool, // comment
+// comment
+int) {
+}
+func _(x bool, // comment
+// comment
+y int) {
+}
+func _(x, // comment
+// comment
+y bool) {
+}
+func _(bool,
+// comment
+int) {
+}
+func _(x bool,
+// comment
+y int) {
+}
+func _(x,
+// comment
+y bool) {
+}
+func _(x, // comment
+y,// comment
+z bool) {
+}
+func _(x, // comment
+	y,// comment
+	z bool) {
+}
+func _(x int,	// comment
+	y float,	// comment
+	z bool) {
+}
diff --git a/libgo/go/go/printer/testdata/empty.golden b/libgo/go/go/printer/testdata/empty.golden
new file mode 100644
index 000000000..a055f4758
--- /dev/null
+++ b/libgo/go/go/printer/testdata/empty.golden
@@ -0,0 +1,5 @@
+// a comment at the beginning of the file
+
+package empty
+
+// a comment at the end of the file
diff --git a/libgo/go/go/printer/testdata/empty.input b/libgo/go/go/printer/testdata/empty.input
new file mode 100644
index 000000000..a055f4758
--- /dev/null
+++ b/libgo/go/go/printer/testdata/empty.input
@@ -0,0 +1,5 @@
+// a comment at the beginning of the file
+
+package empty
+
+// a comment at the end of the file
diff --git a/libgo/go/go/printer/testdata/expressions.golden b/libgo/go/go/printer/testdata/expressions.golden
new file mode 100644
index 000000000..882c7624c
--- /dev/null
+++ b/libgo/go/go/printer/testdata/expressions.golden
@@ -0,0 +1,554 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package expressions
+
+type T struct {
+	x, y, z int
+}
+
+var (
+	a, b, c, d, e						int
+	under_bar						int
+	longIdentifier1, longIdentifier2, longIdentifier3	int
+	t0, t1, t2						T
+	s							string
+	p							*int
+)
+
+
+func _() {
+	// no spaces around simple or parenthesized expressions
+	_ = (a + 0)
+	_ = a + b
+	_ = a + b + c
+	_ = a + b - c
+	_ = a - b - c
+	_ = a + (b * c)
+	_ = a + (b / c)
+	_ = a - (b % c)
+	_ = 1 + a
+	_ = a + 1
+	_ = a + b + 1
+	_ = s[a]
+	_ = s[a:]
+	_ = s[:b]
+	_ = s[1:2]
+	_ = s[a:b]
+	_ = s[0:len(s)]
+	_ = s[0] << 1
+	_ = (s[0] << 1) & 0xf
+	_ = s[0]<<2 | s[1]>>4
+	_ = "foo" + s
+	_ = s + "foo"
+	_ = 'a' + 'b'
+	_ = len(s) / 2
+	_ = len(t0.x) / a
+
+	// spaces around expressions of different precedence or expressions containing spaces
+	_ = a + -b
+	_ = a - ^b
+	_ = a / *p
+	_ = a + b*c
+	_ = 1 + b*c
+	_ = a + 2*c
+	_ = a + c*2
+	_ = 1 + 2*3
+	_ = s[1 : 2*3]
+	_ = s[a : b-c]
+	_ = s[0:]
+	_ = s[a+b]
+	_ = s[:b-c]
+	_ = s[a+b:]
+	_ = a[a< b
+	_ = a >= b
+	_ = a < b
+	_ = a <= b
+	_ = a < b && c > d
+	_ = a < b || c > d
+
+	// spaces around "long" operands
+	_ = a + longIdentifier1
+	_ = longIdentifier1 + a
+	_ = longIdentifier1 + longIdentifier2*longIdentifier3
+	_ = s + "a longer string"
+
+	// some selected cases
+	_ = a + t0.x
+	_ = a + t0.x + t1.x*t2.x
+	_ = a + b + c + d + e + 2*3
+	_ = a + b + c + 2*3 + d + e
+	_ = (a + b + c) * 2
+	_ = a - b + c - d + (a + b + c) + d&e
+	_ = under_bar - 1
+	_ = Open(dpath+"/file", O_WRONLY|O_CREAT, 0666)
+	_ = int(c0&_Mask4)<<18 | int(c1&_Maskx)<<12 | int(c2&_Maskx)<<6 | int(c3&_Maskx)
+}
+
+
+func _() {
+	a + b
+	a + b + c
+	a + b*c
+	a + (b * c)
+	(a + b) * c
+	a + (b * c * d)
+	a + (b*c + d)
+
+	1 << x
+	-1 << x
+	1<>4
+
+	b.buf = b.buf[0 : b.off+m+n]
+	b.buf = b.buf[0 : b.off+m*n]
+	f(b.buf[0 : b.off+m+n])
+
+	signed += ' ' * 8
+	tw.octal(header[148:155], chksum)
+
+	x > 0 && i >= 0
+
+	x1, x0 := x>>w2, x&m2
+	z0 = t1<>w2) >> w2
+	q1, r1 := x1/d1, x1%d1
+	r1 = r1*b2 | x0>>w2
+	x1 = (x1 << z) | (x0 >> (uint(w) - z))
+	x1 = x1<>(uint(w)-z)
+
+	buf[0 : len(buf)+1]
+	buf[0 : n+1]
+
+	a, b = b, a
+	a = b + c
+	a = b*c + d
+	a*b + c
+	a - b - c
+	a - (b - c)
+	a - b*c
+	a - (b * c)
+	a * b / c
+	a / *b
+	x[a|^b]
+	x[a / *b]
+	a & ^b
+	a + +b
+	a - -b
+	x[a*-b]
+	x[a + +b]
+	x ^ y ^ z
+	b[a>>24] ^ b[(a>>16)&0xFF] ^ b[(a>>8)&0xFF] ^ b[a&0xFF]
+	len(longVariableName) * 2
+
+	token(matchType + xlength< multiLine result
+}
+
+
+func _() {
+	// do not modify literals
+	_ = "tab1	tab2	tab3	end"	// string contains 3 tabs
+	_ = "tab1 tab2 tab3 end"	// same string with 3 blanks - may be unaligned because editors see tabs in strings
+	_ = ""				// this comment should be aligned with the one on the previous line
+	_ = ``
+	_ = `
+`
+	_ = `foo
+		bar`
+	_ = `three spaces before the end of the line starting here:   
+they must not be removed`
+}
+
+
+func _() {
+	// one-line function literals (body is on a single line)
+	_ = func() {}
+	_ = func() int { return 0 }
+	_ = func(x, y int) bool { m := (x + y) / 2; return m < 0 }
+
+	// multi-line function literals (body is not on one line)
+	_ = func() {
+	}
+	_ = func() int {
+		return 0
+	}
+	_ = func(x, y int) bool {
+		m := (x + y) / 2
+		return x < y
+	}
+
+	f(func() {
+	})
+	f(func() int {
+		return 0
+	})
+	f(func(x, y int) bool {
+		m := (x + y) / 2
+		return x < y
+	})
+}
+
+
+func _() {
+	_ = [][]int{
+		[]int{1},
+		[]int{1, 2},
+		[]int{1, 2, 3},
+	}
+	_ = [][]int{
+		{1},
+		[]int{1, 2},
+		[]int{1, 2, 3},
+	}
+	_ = [][]int{
+		{1},
+		{1, 2},
+		{1, 2, 3},
+	}
+	_ = [][]int{{1}, {1, 2}, {1, 2, 3}}
+}
+
+
+// various multi-line expressions
+func _() {
+	// do not add extra indentation to multi-line string lists
+	_ = "foo" + "bar"
+	_ = "foo" +
+		"bar" +
+		"bah"
+	_ = []string{
+		"abc" +
+			"def",
+		"foo" +
+			"bar",
+	}
+}
+
+
+const _ = F1 +
+	`string = "%s";` +
+	`ptr = *;` +
+	`datafmt.T2 = s ["-" p "-"];`
+
+
+const _ = `datafmt "datafmt";` +
+	`default = "%v";` +
+	`array = *;` +
+	`datafmt.T3 = s  {" " a a / ","};`
+
+
+const _ = `datafmt "datafmt";` +
+	`default = "%v";` +
+	`array = *;` +
+	`datafmt.T3 = s  {" " a a / ","};`
+
+
+func _() {
+	_ = F1 +
+		`string = "%s";` +
+		`ptr = *;` +
+		`datafmt.T2 = s ["-" p "-"];`
+
+	_ =
+		`datafmt "datafmt";` +
+			`default = "%v";` +
+			`array = *;` +
+			`datafmt.T3 = s  {" " a a / ","};`
+
+	_ = `datafmt "datafmt";` +
+		`default = "%v";` +
+		`array = *;` +
+		`datafmt.T3 = s  {" " a a / ","};`
+}
+
+
+func _() {
+	// respect source lines in multi-line expressions
+	_ = a +
+		b +
+		c
+	_ = a < b ||
+		b < a
+	_ = "933262154439441526816992388562667004907159682643816214685929" +
+		"638952175999932299156089414639761565182862536979208272237582" +
+		"51185210916864000000000000000000000000"	// 100!
+	_ = "170141183460469231731687303715884105727"	// prime
+}
+
+
+// Alignment after overlong lines
+const (
+	_	= "991"
+	_	= "2432902008176640000"	// 20!
+	_	= "933262154439441526816992388562667004907159682643816214685929" +
+		"638952175999932299156089414639761565182862536979208272237582" +
+		"51185210916864000000000000000000000000"	// 100!
+	_	= "170141183460469231731687303715884105727"	// prime
+)
+
+
+// Correct placement of operators and comments in multi-line expressions
+func _() {
+	_ = a +	// comment
+		b +	// comment
+		c
+	_ = "a" +
+		"b" +	// comment
+		"c"
+	_ = "ba0408" + "7265717569726564"	// field 71, encoding 2, string "required"
+}
+
+
+// Correct placement of terminating comma/closing parentheses in multi-line calls.
+func _() {
+	f(1,
+		2,
+		3)
+	f(1,
+		2,
+		3,
+	)
+	f(1,
+		2,
+		3)	// comment
+	f(1,
+		2,
+		3,	// comment
+	)
+	f(1,
+		2,
+		3)	// comment
+	f(1,
+		2,
+		3,	// comment
+	)
+}
+
+
+// Align comments in multi-line lists of single-line expressions.
+var txpix = [NCOL]draw.Color{
+	draw.Yellow,		// yellow
+	draw.Cyan,		// cyan
+	draw.Green,		// lime green
+	draw.GreyBlue,		// slate
+	draw.Red,		/* red */
+	draw.GreyGreen,		/* olive green */
+	draw.Blue,		/* blue */
+	draw.Color(0xFF55AAFF),	/* pink */
+	draw.Color(0xFFAAFFFF),	/* lavender */
+	draw.Color(0xBB005DFF),	/* maroon */
+}
+
+
+func same(t, u *Time) bool {
+	// respect source lines in multi-line expressions
+	return t.Year == u.Year &&
+		t.Month == u.Month &&
+		t.Day == u.Day &&
+		t.Hour == u.Hour &&
+		t.Minute == u.Minute &&
+		t.Second == u.Second &&
+		t.Weekday == u.Weekday &&
+		t.ZoneOffset == u.ZoneOffset &&
+		t.Zone == u.Zone
+}
+
+
+func (p *parser) charClass() {
+	// respect source lines in multi-line expressions
+	if cc.negate && len(cc.ranges) == 2 &&
+		cc.ranges[0] == '\n' && cc.ranges[1] == '\n' {
+		nl := new(_NotNl)
+		p.re.add(nl)
+	}
+}
+
+
+func addState(s []state, inst instr, match []int) {
+	// handle comments correctly in multi-line expressions
+	for i := 0; i < l; i++ {
+		if s[i].inst.index() == index &&	// same instruction
+			s[i].match[0] < pos {	// earlier match already going; leftmost wins
+			return s
+		}
+	}
+}
+
+func (self *T) foo(x int) *T	{ return self }
+
+func _()	{ module.Func1().Func2() }
+
+func _() {
+	_ = new(T).
+		foo(1).
+		foo(2).
+		foo(3)
+
+	_ = new(T).
+		foo(1).
+		foo(2).	// inline comments
+		foo(3)
+
+	_ = new(T).foo(1).foo(2).foo(3)
+
+	// handle multiline argument list correctly
+	_ = new(T).
+		foo(
+			1).
+		foo(2)
+
+	_ = new(T).foo(
+		1).foo(2)
+
+	_ = Array[3+
+		4]
+
+	_ = Method(1, 2,
+		3)
+
+	_ = new(T).
+		foo().
+		bar().(*Type)
+
+	_ = new(T).
+		foo().
+		bar().(*Type).
+		baz()
+
+	_ = new(T).
+		foo().
+		bar()["idx"]
+
+	_ = new(T).
+		foo().
+		bar()["idx"].
+		baz()
+
+	_ = new(T).
+		foo().
+		bar()[1:2]
+
+	_ = new(T).
+		foo().
+		bar()[1:2].
+		baz()
+
+	_ = new(T).
+		Field.
+		Array[3+
+			4].
+		Table["foo"].
+		Blob.(*Type).
+		Slices[1:4].
+		Method(1, 2,
+			3).
+		Thingy
+
+	_ = a.b.c
+	_ = a.
+		b.
+		c
+	_ = a.b().c
+	_ = a.
+		b().
+		c
+	_ = a.b[0].c
+	_ = a.
+		b[0].
+		c
+	_ = a.b[0:].c
+	_ = a.
+		b[0:].
+		c
+	_ = a.b.(T).c
+	_ = a.
+		b.(T).
+		c
+}
diff --git a/libgo/go/go/printer/testdata/expressions.input b/libgo/go/go/printer/testdata/expressions.input
new file mode 100644
index 000000000..647706b09
--- /dev/null
+++ b/libgo/go/go/printer/testdata/expressions.input
@@ -0,0 +1,548 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package expressions
+
+type T struct {
+	x, y, z int
+}
+
+var (
+	a, b, c, d, e int
+	under_bar int
+	longIdentifier1, longIdentifier2, longIdentifier3 int
+	t0, t1, t2 T
+	s string
+	p *int
+)
+
+
+func _() {
+	// no spaces around simple or parenthesized expressions
+	_ = (a+0)
+	_ = a+b
+	_ = a+b+c
+	_ = a+b-c
+	_ = a-b-c
+	_ = a+(b*c)
+	_ = a+(b/c)
+	_ = a-(b%c)
+	_ = 1+a
+	_ = a+1
+	_ = a+b+1
+	_ = s[a]
+	_ = s[a:]
+	_ = s[:b]
+	_ = s[1:2]
+	_ = s[a:b]
+	_ = s[0:len(s)]
+	_ = s[0]<<1
+	_ = (s[0]<<1)&0xf
+	_ = s[0] << 2 | s[1] >> 4
+	_ = "foo"+s
+	_ = s+"foo"
+	_ = 'a'+'b'
+	_ = len(s)/2
+	_ = len(t0.x)/a
+
+	// spaces around expressions of different precedence or expressions containing spaces
+	_ = a + -b
+	_ = a - ^b
+	_ = a / *p
+	_ = a + b*c
+	_ = 1 + b*c
+	_ = a + 2*c
+	_ = a + c*2
+	_ = 1 + 2*3
+	_ = s[1 : 2*3]
+	_ = s[a : b-c]
+	_ = s[0:]
+	_ = s[a+b]
+	_ = s[: b-c]
+	_ = s[a+b :]
+	_ = a[a< b
+	_ = a >= b
+	_ = a < b
+	_ = a <= b
+	_ = a < b && c > d
+	_ = a < b || c > d
+
+	// spaces around "long" operands
+	_ = a + longIdentifier1
+	_ = longIdentifier1 + a
+	_ = longIdentifier1 + longIdentifier2 * longIdentifier3
+	_ = s + "a longer string"
+
+	// some selected cases
+	_ = a + t0.x
+	_ = a + t0.x + t1.x * t2.x
+	_ = a + b + c + d + e + 2*3
+	_ = a + b + c + 2*3 + d + e
+	_ = (a+b+c)*2
+	_ = a - b + c - d + (a+b+c) + d&e
+	_ = under_bar-1
+	_ = Open(dpath + "/file", O_WRONLY | O_CREAT, 0666)
+	_ = int(c0&_Mask4)<<18 | int(c1&_Maskx)<<12 | int(c2&_Maskx)<<6 | int(c3&_Maskx)
+}
+
+
+func _() {
+	a+b
+	a+b+c
+	a+b*c
+	a+(b*c)
+	(a+b)*c
+	a+(b*c*d)
+	a+(b*c+d)
+
+	1<>4
+
+	b.buf = b.buf[0:b.off+m+n]
+	b.buf = b.buf[0:b.off+m*n]
+	f(b.buf[0:b.off+m+n])
+
+	signed += ' '*8
+	tw.octal(header[148:155], chksum)
+
+	x > 0 && i >= 0
+
+	x1, x0 := x>>w2, x&m2
+	z0 = t1<>w2)>>w2
+	q1, r1 := x1/d1, x1%d1
+	r1 = r1*b2 | x0>>w2
+	x1 = (x1<>(uint(w)-z))
+	x1 = x1<>(uint(w)-z)
+
+	buf[0:len(buf)+1]
+	buf[0:n+1]
+
+	a,b = b,a
+	a = b+c
+	a = b*c+d
+	a*b+c
+	a-b-c
+	a-(b-c)
+	a-b*c
+	a-(b*c)
+	a*b/c
+	a/ *b
+	x[a|^b]
+	x[a/ *b]
+	a& ^b
+	a+ +b
+	a- -b
+	x[a*-b]
+	x[a+ +b]
+	x^y^z
+	b[a>>24] ^ b[(a>>16)&0xFF] ^ b[(a>>8)&0xFF] ^ b[a&0xFF]
+	len(longVariableName)*2
+
+	token(matchType + xlength< multiLine result
+}
+
+
+func _() {
+	// do not modify literals
+	_ = "tab1	tab2	tab3	end"  // string contains 3 tabs
+	_ = "tab1 tab2 tab3 end"  // same string with 3 blanks - may be unaligned because editors see tabs in strings
+	_ = ""  // this comment should be aligned with the one on the previous line
+	_ = ``
+	_ = `
+`
+_ = `foo
+		bar`
+	_ = `three spaces before the end of the line starting here:   
+they must not be removed`
+}
+
+
+func _() {
+	// one-line function literals (body is on a single line)
+	_ = func() {}
+	_ = func() int { return 0 }
+	_ = func(x, y int) bool { m := (x+y)/2; return m < 0 }
+
+	// multi-line function literals (body is not on one line)
+	_ = func() {
+	}
+	_ = func() int {
+		return 0
+	}
+	_ = func(x, y int) bool {
+		m := (x+y)/2; return x < y }
+
+	f(func() {
+	})
+	f(func() int {
+		return 0
+	})
+	f(func(x, y int) bool {
+		m := (x+y)/2; return x < y })
+}
+
+
+func _() {
+	_ = [][]int {
+		[]int{1},
+		[]int{1, 2},
+		[]int{1, 2, 3},
+	}
+	_ = [][]int {
+		{1},
+		[]int{1, 2},
+		[]int{1, 2, 3},
+	}
+	_ = [][]int {
+		{1},
+		{1, 2},
+		{1, 2, 3},
+	}
+	_ = [][]int {{1}, {1, 2}, {1, 2, 3}}
+}
+
+
+// various multi-line expressions
+func _() {
+	// do not add extra indentation to multi-line string lists
+	_ = "foo" + "bar"
+	_ = "foo" +
+	"bar" +
+	"bah"
+	_ = []string {
+		"abc" +
+		"def",
+		"foo" +
+		"bar",
+	}
+}
+
+
+const _ = F1 +
+	`string = "%s";` +
+	`ptr = *;` +
+	`datafmt.T2 = s ["-" p "-"];`
+
+
+const _ =
+	`datafmt "datafmt";` +
+	`default = "%v";` +
+	`array = *;` +
+	`datafmt.T3 = s  {" " a a / ","};`
+
+
+const _ = `datafmt "datafmt";` +
+`default = "%v";` +
+`array = *;` +
+`datafmt.T3 = s  {" " a a / ","};`
+
+
+func _() {
+	_ = F1 +
+		`string = "%s";` +
+		`ptr = *;` +
+		`datafmt.T2 = s ["-" p "-"];`
+
+	_ =
+		`datafmt "datafmt";` +
+		`default = "%v";` +
+		`array = *;` +
+		`datafmt.T3 = s  {" " a a / ","};`
+
+	_ = `datafmt "datafmt";` +
+	`default = "%v";` +
+	`array = *;` +
+	`datafmt.T3 = s  {" " a a / ","};`
+}
+
+
+func _() {
+	// respect source lines in multi-line expressions
+	_ = a+
+	b+
+	c
+	_ = a < b ||
+		b < a
+	_ = "933262154439441526816992388562667004907159682643816214685929" +
+	"638952175999932299156089414639761565182862536979208272237582" +
+	"51185210916864000000000000000000000000"  // 100!
+	_ = "170141183460469231731687303715884105727"  // prime
+}
+
+
+// Alignment after overlong lines
+const (
+	_ = "991"
+	_ = "2432902008176640000"  // 20!
+	_ = "933262154439441526816992388562667004907159682643816214685929" +
+	"638952175999932299156089414639761565182862536979208272237582" +
+	"51185210916864000000000000000000000000"  // 100!
+	_ = "170141183460469231731687303715884105727"  // prime
+)
+
+
+// Correct placement of operators and comments in multi-line expressions
+func _() {
+	_ = a +  // comment
+		b +  // comment
+		c
+	_ = "a"	+
+		"b" +	// comment
+		"c"
+	_ = "ba0408" + "7265717569726564"     // field 71, encoding 2, string "required"
+}
+
+
+// Correct placement of terminating comma/closing parentheses in multi-line calls.
+func _() {
+	f(1,
+		2,
+		3)
+	f(1,
+		2,
+		3,
+	)
+	f(1,
+		2,
+		3)  // comment
+	f(1,
+		2,
+		3,  // comment
+	)
+	f(1,
+		2,
+		3)// comment
+	f(1,
+		2,
+		3,// comment
+	)
+}
+
+
+// Align comments in multi-line lists of single-line expressions.
+var txpix = [NCOL]draw.Color{
+	draw.Yellow, // yellow
+	draw.Cyan, // cyan
+	draw.Green, // lime green
+	draw.GreyBlue, // slate
+	draw.Red, /* red */
+	draw.GreyGreen, /* olive green */
+	draw.Blue, /* blue */
+	draw.Color(0xFF55AAFF), /* pink */
+	draw.Color(0xFFAAFFFF), /* lavender */
+	draw.Color(0xBB005DFF), /* maroon */
+}
+
+
+func same(t, u *Time) bool {
+	// respect source lines in multi-line expressions
+	return t.Year == u.Year &&
+		t.Month == u.Month &&
+		t.Day == u.Day &&
+		t.Hour == u.Hour &&
+		t.Minute == u.Minute &&
+		t.Second == u.Second &&
+		t.Weekday == u.Weekday &&
+		t.ZoneOffset == u.ZoneOffset &&
+		t.Zone == u.Zone
+}
+
+
+func (p *parser) charClass() {
+	// respect source lines in multi-line expressions
+	if cc.negate && len(cc.ranges) == 2 &&
+		cc.ranges[0] == '\n' && cc.ranges[1] == '\n' {
+		nl := new(_NotNl)
+		p.re.add(nl)
+	}
+}
+
+
+func addState(s []state, inst instr, match []int) {
+	// handle comments correctly in multi-line expressions
+	for i := 0; i < l; i++ {
+		if s[i].inst.index() == index && // same instruction
+		   s[i].match[0] < pos {	// earlier match already going; leftmost wins
+		   	return s
+		 }
+	}
+}
+
+func (self *T) foo(x int) *T { return self }
+
+func _() { module.Func1().Func2() }
+
+func _() {
+	_ = new(T).
+		foo(1).
+			foo(2).
+		foo(3)
+
+	_ = new(T).
+	foo(1).
+	foo(2). // inline comments
+	foo(3)
+
+	_ = new(T).foo(1).foo(2).foo(3)
+
+	// handle multiline argument list correctly
+	_ = new(T).
+	foo(
+		1).
+		foo(2)
+
+	_ = new(T).foo(
+		1).foo(2)
+
+	_ = Array[3 +
+4]
+
+	_ = Method(1, 2,
+		3)
+
+	_ = new(T).
+   foo().
+   bar() . (*Type)
+
+	_ = new(T).
+foo().
+bar().(*Type).
+baz()
+
+	_ = new(T).
+	foo().
+	bar()["idx"]
+
+	_ = new(T).
+	foo().
+	bar()["idx"]	.
+	baz()
+
+	_ = new(T).
+	foo().
+	bar()[1:2]
+
+	_ = new(T).
+	foo().
+	bar()[1:2].
+	baz()
+
+	_ = new(T).
+		Field.
+		Array[3+
+       		4].
+		Table ["foo"].
+		Blob. (*Type).
+	Slices[1:4].
+	Method(1, 2,
+	3).
+		Thingy
+
+	_ = a.b.c
+	_ = a.
+	b.
+	c
+	_ = a.b().c
+	_ = a.
+	b().
+	c
+	_ = a.b[0].c
+	_ = a.
+	b[0].
+	c
+	_ = a.b[0:].c
+	_ = a.
+	b[0:].
+	c
+	_ = a.b.(T).c
+	_ = a.
+	b.
+	(T).
+	c
+}
diff --git a/libgo/go/go/printer/testdata/expressions.raw b/libgo/go/go/printer/testdata/expressions.raw
new file mode 100644
index 000000000..62be00cc3
--- /dev/null
+++ b/libgo/go/go/printer/testdata/expressions.raw
@@ -0,0 +1,554 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package expressions
+
+type T struct {
+	x, y, z int
+}
+
+var (
+	a, b, c, d, e	int
+	under_bar	int
+	longIdentifier1, longIdentifier2, longIdentifier3	int
+	t0, t1, t2	T
+	s	string
+	p	*int
+)
+
+
+func _() {
+	// no spaces around simple or parenthesized expressions
+	_ = (a + 0)
+	_ = a + b
+	_ = a + b + c
+	_ = a + b - c
+	_ = a - b - c
+	_ = a + (b * c)
+	_ = a + (b / c)
+	_ = a - (b % c)
+	_ = 1 + a
+	_ = a + 1
+	_ = a + b + 1
+	_ = s[a]
+	_ = s[a:]
+	_ = s[:b]
+	_ = s[1:2]
+	_ = s[a:b]
+	_ = s[0:len(s)]
+	_ = s[0] << 1
+	_ = (s[0] << 1) & 0xf
+	_ = s[0]<<2 | s[1]>>4
+	_ = "foo" + s
+	_ = s + "foo"
+	_ = 'a' + 'b'
+	_ = len(s) / 2
+	_ = len(t0.x) / a
+
+	// spaces around expressions of different precedence or expressions containing spaces
+	_ = a + -b
+	_ = a - ^b
+	_ = a / *p
+	_ = a + b*c
+	_ = 1 + b*c
+	_ = a + 2*c
+	_ = a + c*2
+	_ = 1 + 2*3
+	_ = s[1 : 2*3]
+	_ = s[a : b-c]
+	_ = s[0:]
+	_ = s[a+b]
+	_ = s[:b-c]
+	_ = s[a+b:]
+	_ = a[a< b
+	_ = a >= b
+	_ = a < b
+	_ = a <= b
+	_ = a < b && c > d
+	_ = a < b || c > d
+
+	// spaces around "long" operands
+	_ = a + longIdentifier1
+	_ = longIdentifier1 + a
+	_ = longIdentifier1 + longIdentifier2*longIdentifier3
+	_ = s + "a longer string"
+
+	// some selected cases
+	_ = a + t0.x
+	_ = a + t0.x + t1.x*t2.x
+	_ = a + b + c + d + e + 2*3
+	_ = a + b + c + 2*3 + d + e
+	_ = (a + b + c) * 2
+	_ = a - b + c - d + (a + b + c) + d&e
+	_ = under_bar - 1
+	_ = Open(dpath+"/file", O_WRONLY|O_CREAT, 0666)
+	_ = int(c0&_Mask4)<<18 | int(c1&_Maskx)<<12 | int(c2&_Maskx)<<6 | int(c3&_Maskx)
+}
+
+
+func _() {
+	a + b
+	a + b + c
+	a + b*c
+	a + (b * c)
+	(a + b) * c
+	a + (b * c * d)
+	a + (b*c + d)
+
+	1 << x
+	-1 << x
+	1<>4
+
+	b.buf = b.buf[0 : b.off+m+n]
+	b.buf = b.buf[0 : b.off+m*n]
+	f(b.buf[0 : b.off+m+n])
+
+	signed += ' ' * 8
+	tw.octal(header[148:155], chksum)
+
+	x > 0 && i >= 0
+
+	x1, x0 := x>>w2, x&m2
+	z0 = t1<>w2) >> w2
+	q1, r1 := x1/d1, x1%d1
+	r1 = r1*b2 | x0>>w2
+	x1 = (x1 << z) | (x0 >> (uint(w) - z))
+	x1 = x1<>(uint(w)-z)
+
+	buf[0 : len(buf)+1]
+	buf[0 : n+1]
+
+	a, b = b, a
+	a = b + c
+	a = b*c + d
+	a*b + c
+	a - b - c
+	a - (b - c)
+	a - b*c
+	a - (b * c)
+	a * b / c
+	a / *b
+	x[a|^b]
+	x[a / *b]
+	a & ^b
+	a + +b
+	a - -b
+	x[a*-b]
+	x[a + +b]
+	x ^ y ^ z
+	b[a>>24] ^ b[(a>>16)&0xFF] ^ b[(a>>8)&0xFF] ^ b[a&0xFF]
+	len(longVariableName) * 2
+
+	token(matchType + xlength< multiLine result
+}
+
+
+func _() {
+	// do not modify literals
+	_ = "tab1	tab2	tab3	end"	// string contains 3 tabs
+	_ = "tab1 tab2 tab3 end"	// same string with 3 blanks - may be unaligned because editors see tabs in strings
+	_ = ""	// this comment should be aligned with the one on the previous line
+	_ = ``
+	_ = `
+`
+	_ = `foo
+		bar`
+	_ = `three spaces before the end of the line starting here:   
+they must not be removed`
+}
+
+
+func _() {
+	// one-line function literals (body is on a single line)
+	_ = func() {}
+	_ = func() int { return 0 }
+	_ = func(x, y int) bool { m := (x + y) / 2; return m < 0 }
+
+	// multi-line function literals (body is not on one line)
+	_ = func() {
+	}
+	_ = func() int {
+		return 0
+	}
+	_ = func(x, y int) bool {
+		m := (x + y) / 2
+		return x < y
+	}
+
+	f(func() {
+	})
+	f(func() int {
+		return 0
+	})
+	f(func(x, y int) bool {
+		m := (x + y) / 2
+		return x < y
+	})
+}
+
+
+func _() {
+	_ = [][]int{
+		[]int{1},
+		[]int{1, 2},
+		[]int{1, 2, 3},
+	}
+	_ = [][]int{
+		{1},
+		[]int{1, 2},
+		[]int{1, 2, 3},
+	}
+	_ = [][]int{
+		{1},
+		{1, 2},
+		{1, 2, 3},
+	}
+	_ = [][]int{{1}, {1, 2}, {1, 2, 3}}
+}
+
+
+// various multi-line expressions
+func _() {
+	// do not add extra indentation to multi-line string lists
+	_ = "foo" + "bar"
+	_ = "foo" +
+		"bar" +
+		"bah"
+	_ = []string{
+		"abc" +
+			"def",
+		"foo" +
+			"bar",
+	}
+}
+
+
+const _ = F1 +
+	`string = "%s";` +
+	`ptr = *;` +
+	`datafmt.T2 = s ["-" p "-"];`
+
+
+const _ = `datafmt "datafmt";` +
+	`default = "%v";` +
+	`array = *;` +
+	`datafmt.T3 = s  {" " a a / ","};`
+
+
+const _ = `datafmt "datafmt";` +
+	`default = "%v";` +
+	`array = *;` +
+	`datafmt.T3 = s  {" " a a / ","};`
+
+
+func _() {
+	_ = F1 +
+		`string = "%s";` +
+		`ptr = *;` +
+		`datafmt.T2 = s ["-" p "-"];`
+
+	_ =
+		`datafmt "datafmt";` +
+			`default = "%v";` +
+			`array = *;` +
+			`datafmt.T3 = s  {" " a a / ","};`
+
+	_ = `datafmt "datafmt";` +
+		`default = "%v";` +
+		`array = *;` +
+		`datafmt.T3 = s  {" " a a / ","};`
+}
+
+
+func _() {
+	// respect source lines in multi-line expressions
+	_ = a +
+		b +
+		c
+	_ = a < b ||
+		b < a
+	_ = "933262154439441526816992388562667004907159682643816214685929" +
+		"638952175999932299156089414639761565182862536979208272237582" +
+		"51185210916864000000000000000000000000"	// 100!
+	_ = "170141183460469231731687303715884105727"	// prime
+}
+
+
+// Alignment after overlong lines
+const (
+	_	= "991"
+	_	= "2432902008176640000"		// 20!
+	_	= "933262154439441526816992388562667004907159682643816214685929" +
+		"638952175999932299156089414639761565182862536979208272237582" +
+		"51185210916864000000000000000000000000"	// 100!
+	_	= "170141183460469231731687303715884105727"		// prime
+)
+
+
+// Correct placement of operators and comments in multi-line expressions
+func _() {
+	_ = a +	// comment
+		b +	// comment
+		c
+	_ = "a" +
+		"b" +	// comment
+		"c"
+	_ = "ba0408" + "7265717569726564"	// field 71, encoding 2, string "required"
+}
+
+
+// Correct placement of terminating comma/closing parentheses in multi-line calls.
+func _() {
+	f(1,
+		2,
+		3)
+	f(1,
+		2,
+		3,
+	)
+	f(1,
+		2,
+		3)	// comment
+	f(1,
+		2,
+		3,	// comment
+	)
+	f(1,
+		2,
+		3)	// comment
+	f(1,
+		2,
+		3,	// comment
+	)
+}
+
+
+// Align comments in multi-line lists of single-line expressions.
+var txpix = [NCOL]draw.Color{
+	draw.Yellow,	// yellow
+	draw.Cyan,	// cyan
+	draw.Green,	// lime green
+	draw.GreyBlue,	// slate
+	draw.Red,	/* red */
+	draw.GreyGreen,	/* olive green */
+	draw.Blue,	/* blue */
+	draw.Color(0xFF55AAFF),	/* pink */
+	draw.Color(0xFFAAFFFF),	/* lavender */
+	draw.Color(0xBB005DFF),	/* maroon */
+}
+
+
+func same(t, u *Time) bool {
+	// respect source lines in multi-line expressions
+	return t.Year == u.Year &&
+		t.Month == u.Month &&
+		t.Day == u.Day &&
+		t.Hour == u.Hour &&
+		t.Minute == u.Minute &&
+		t.Second == u.Second &&
+		t.Weekday == u.Weekday &&
+		t.ZoneOffset == u.ZoneOffset &&
+		t.Zone == u.Zone
+}
+
+
+func (p *parser) charClass() {
+	// respect source lines in multi-line expressions
+	if cc.negate && len(cc.ranges) == 2 &&
+		cc.ranges[0] == '\n' && cc.ranges[1] == '\n' {
+		nl := new(_NotNl)
+		p.re.add(nl)
+	}
+}
+
+
+func addState(s []state, inst instr, match []int) {
+	// handle comments correctly in multi-line expressions
+	for i := 0; i < l; i++ {
+		if s[i].inst.index() == index &&	// same instruction
+			s[i].match[0] < pos {	// earlier match already going; leftmost wins
+			return s
+		}
+	}
+}
+
+func (self *T) foo(x int) *T	{ return self }
+
+func _()	{ module.Func1().Func2() }
+
+func _() {
+	_ = new(T).
+		foo(1).
+		foo(2).
+		foo(3)
+
+	_ = new(T).
+		foo(1).
+		foo(2).	// inline comments
+		foo(3)
+
+	_ = new(T).foo(1).foo(2).foo(3)
+
+	// handle multiline argument list correctly
+	_ = new(T).
+		foo(
+			1).
+		foo(2)
+
+	_ = new(T).foo(
+		1).foo(2)
+
+	_ = Array[3+
+		4]
+
+	_ = Method(1, 2,
+		3)
+
+	_ = new(T).
+		foo().
+		bar().(*Type)
+
+	_ = new(T).
+		foo().
+		bar().(*Type).
+		baz()
+
+	_ = new(T).
+		foo().
+		bar()["idx"]
+
+	_ = new(T).
+		foo().
+		bar()["idx"].
+		baz()
+
+	_ = new(T).
+		foo().
+		bar()[1:2]
+
+	_ = new(T).
+		foo().
+		bar()[1:2].
+		baz()
+
+	_ = new(T).
+		Field.
+		Array[3+
+			4].
+		Table["foo"].
+		Blob.(*Type).
+		Slices[1:4].
+		Method(1, 2,
+			3).
+		Thingy
+
+	_ = a.b.c
+	_ = a.
+		b.
+		c
+	_ = a.b().c
+	_ = a.
+		b().
+		c
+	_ = a.b[0].c
+	_ = a.
+		b[0].
+		c
+	_ = a.b[0:].c
+	_ = a.
+		b[0:].
+		c
+	_ = a.b.(T).c
+	_ = a.
+		b.(T).
+		c
+}
diff --git a/libgo/go/go/printer/testdata/linebreaks.golden b/libgo/go/go/printer/testdata/linebreaks.golden
new file mode 100644
index 000000000..be780da67
--- /dev/null
+++ b/libgo/go/go/printer/testdata/linebreaks.golden
@@ -0,0 +1,223 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package linebreaks
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+type writerTestEntry struct {
+	header		*Header
+	contents	string
+}
+
+type writerTest struct {
+	file	string	// filename of expected output
+	entries	[]*writerTestEntry
+}
+
+var writerTests = []*writerTest{
+	&writerTest{
+		file:	"testdata/writer.tar",
+		entries: []*writerTestEntry{
+			&writerTestEntry{
+				header: &Header{
+					Name:		"small.txt",
+					Mode:		0640,
+					Uid:		73025,
+					Gid:		5000,
+					Size:		5,
+					Mtime:		1246508266,
+					Typeflag:	'0',
+					Uname:		"dsymonds",
+					Gname:		"eng",
+				},
+				contents:	"Kilts",
+			},
+			&writerTestEntry{
+				header: &Header{
+					Name:		"small2.txt",
+					Mode:		0640,
+					Uid:		73025,
+					Gid:		5000,
+					Size:		11,
+					Mtime:		1245217492,
+					Typeflag:	'0',
+					Uname:		"dsymonds",
+					Gname:		"eng",
+				},
+				contents:	"Google.com\n",
+			},
+		},
+	},
+	// The truncated test file was produced using these commands:
+	//   dd if=/dev/zero bs=1048576 count=16384 > /tmp/16gig.txt
+	//   tar -b 1 -c -f- /tmp/16gig.txt | dd bs=512 count=8 > writer-big.tar
+	&writerTest{
+		file:	"testdata/writer-big.tar",
+		entries: []*writerTestEntry{
+			&writerTestEntry{
+				header: &Header{
+					Name:		"tmp/16gig.txt",
+					Mode:		0640,
+					Uid:		73025,
+					Gid:		5000,
+					Size:		16 << 30,
+					Mtime:		1254699560,
+					Typeflag:	'0',
+					Uname:		"dsymonds",
+					Gname:		"eng",
+				},
+				// no contents
+			},
+		},
+	},
+}
+
+type untarTest struct {
+	file	string
+	headers	[]*Header
+}
+
+var untarTests = []*untarTest{
+	&untarTest{
+		file:	"testdata/gnu.tar",
+		headers: []*Header{
+			&Header{
+				Name:		"small.txt",
+				Mode:		0640,
+				Uid:		73025,
+				Gid:		5000,
+				Size:		5,
+				Mtime:		1244428340,
+				Typeflag:	'0',
+				Uname:		"dsymonds",
+				Gname:		"eng",
+			},
+			&Header{
+				Name:		"small2.txt",
+				Mode:		0640,
+				Uid:		73025,
+				Gid:		5000,
+				Size:		11,
+				Mtime:		1244436044,
+				Typeflag:	'0',
+				Uname:		"dsymonds",
+				Gname:		"eng",
+			},
+		},
+	},
+	&untarTest{
+		file:	"testdata/star.tar",
+		headers: []*Header{
+			&Header{
+				Name:		"small.txt",
+				Mode:		0640,
+				Uid:		73025,
+				Gid:		5000,
+				Size:		5,
+				Mtime:		1244592783,
+				Typeflag:	'0',
+				Uname:		"dsymonds",
+				Gname:		"eng",
+				Atime:		1244592783,
+				Ctime:		1244592783,
+			},
+			&Header{
+				Name:		"small2.txt",
+				Mode:		0640,
+				Uid:		73025,
+				Gid:		5000,
+				Size:		11,
+				Mtime:		1244592783,
+				Typeflag:	'0',
+				Uname:		"dsymonds",
+				Gname:		"eng",
+				Atime:		1244592783,
+				Ctime:		1244592783,
+			},
+		},
+	},
+	&untarTest{
+		file:	"testdata/v7.tar",
+		headers: []*Header{
+			&Header{
+				Name:		"small.txt",
+				Mode:		0444,
+				Uid:		73025,
+				Gid:		5000,
+				Size:		5,
+				Mtime:		1244593104,
+				Typeflag:	'\x00',
+			},
+			&Header{
+				Name:		"small2.txt",
+				Mode:		0444,
+				Uid:		73025,
+				Gid:		5000,
+				Size:		11,
+				Mtime:		1244593104,
+				Typeflag:	'\x00',
+			},
+		},
+	},
+}
+
+var facts = map[int]string{
+	0:	"1",
+	1:	"1",
+	2:	"2",
+	10:	"3628800",
+	20:	"2432902008176640000",
+	100: "933262154439441526816992388562667004907159682643816214685929" +
+		"638952175999932299156089414639761565182862536979208272237582" +
+		"51185210916864000000000000000000000000",
+}
+
+func usage() {
+	fmt.Fprintf(os.Stderr,
+		// TODO(gri): the 2nd string of this string list should not be indented
+		"usage: godoc package [name ...]\n"+
+			"	godoc -http=:6060\n")
+	flag.PrintDefaults()
+	os.Exit(2)
+}
+
+func TestReader(t *testing.T) {
+testLoop:
+	for i, test := range untarTests {
+		f, err := os.Open(test.file, os.O_RDONLY, 0444)
+		if err != nil {
+			t.Errorf("test %d: Unexpected error: %v", i, err)
+			continue
+		}
+		tr := NewReader(f)
+		for j, header := range test.headers {
+			hdr, err := tr.Next()
+			if err != nil || hdr == nil {
+				t.Errorf("test %d, entry %d: Didn't get entry: %v", i, j, err)
+				f.Close()
+				continue testLoop
+			}
+			if !reflect.DeepEqual(hdr, header) {
+				t.Errorf("test %d, entry %d: Incorrect header:\nhave %+v\nwant %+v",
+					i, j, *hdr, *header)
+			}
+		}
+		hdr, err := tr.Next()
+		if hdr != nil || err != nil {
+			t.Errorf("test %d: Unexpected entry or error: hdr=%v err=%v", i, err)
+		}
+		f.Close()
+	}
+}
+
+// There should be exactly one linebreak after this comment.
diff --git a/libgo/go/go/printer/testdata/linebreaks.input b/libgo/go/go/printer/testdata/linebreaks.input
new file mode 100644
index 000000000..457b491e6
--- /dev/null
+++ b/libgo/go/go/printer/testdata/linebreaks.input
@@ -0,0 +1,223 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package linebreaks
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+type writerTestEntry struct {
+	header *Header
+	contents string
+}
+
+type writerTest struct {
+	file string  // filename of expected output
+	entries []*writerTestEntry
+}
+
+var writerTests = []*writerTest{
+	&writerTest{
+		file: "testdata/writer.tar",
+		entries: []*writerTestEntry{
+			&writerTestEntry{
+				header: &Header{
+					Name: "small.txt",
+					Mode: 0640,
+					Uid: 73025,
+					Gid: 5000,
+					Size: 5,
+					Mtime: 1246508266,
+					Typeflag: '0',
+					Uname: "dsymonds",
+					Gname: "eng",
+				},
+				contents: "Kilts",
+			},
+			&writerTestEntry{
+				header: &Header{
+					Name: "small2.txt",
+					Mode: 0640,
+					Uid: 73025,
+					Gid: 5000,
+					Size: 11,
+					Mtime: 1245217492,
+					Typeflag: '0',
+					Uname: "dsymonds",
+					Gname: "eng",
+				},
+				contents: "Google.com\n",
+			},
+		},
+	},
+	// The truncated test file was produced using these commands:
+	//   dd if=/dev/zero bs=1048576 count=16384 > /tmp/16gig.txt
+	//   tar -b 1 -c -f- /tmp/16gig.txt | dd bs=512 count=8 > writer-big.tar
+	&writerTest{
+		file: "testdata/writer-big.tar",
+		entries: []*writerTestEntry{
+			&writerTestEntry{
+				header: &Header{
+					Name: "tmp/16gig.txt",
+					Mode: 0640,
+					Uid: 73025,
+					Gid: 5000,
+					Size: 16 << 30,
+					Mtime: 1254699560,
+					Typeflag: '0',
+					Uname: "dsymonds",
+					Gname: "eng",
+				},
+				// no contents
+			},
+		},
+	},
+}
+
+type untarTest struct {
+	file string
+	headers []*Header
+}
+
+var untarTests = []*untarTest{
+	&untarTest{
+		file: "testdata/gnu.tar",
+		headers: []*Header{
+			&Header{
+				Name: "small.txt",
+				Mode: 0640,
+				Uid: 73025,
+				Gid: 5000,
+				Size: 5,
+				Mtime: 1244428340,
+				Typeflag: '0',
+				Uname: "dsymonds",
+				Gname: "eng",
+			},
+			&Header{
+				Name: "small2.txt",
+				Mode: 0640,
+				Uid: 73025,
+				Gid: 5000,
+				Size: 11,
+				Mtime: 1244436044,
+				Typeflag: '0',
+				Uname: "dsymonds",
+				Gname: "eng",
+			},
+		},
+	},
+	&untarTest{
+		file: "testdata/star.tar",
+		headers: []*Header{
+			&Header{
+				Name: "small.txt",
+				Mode: 0640,
+				Uid: 73025,
+				Gid: 5000,
+				Size: 5,
+				Mtime: 1244592783,
+				Typeflag: '0',
+				Uname: "dsymonds",
+				Gname: "eng",
+				Atime: 1244592783,
+				Ctime: 1244592783,
+			},
+			&Header{
+				Name: "small2.txt",
+				Mode: 0640,
+				Uid: 73025,
+				Gid: 5000,
+				Size: 11,
+				Mtime: 1244592783,
+				Typeflag: '0',
+				Uname: "dsymonds",
+				Gname: "eng",
+				Atime: 1244592783,
+				Ctime: 1244592783,
+			},
+		},
+	},
+	&untarTest{
+		file: "testdata/v7.tar",
+		headers: []*Header{
+			&Header{
+				Name: "small.txt",
+				Mode: 0444,
+				Uid: 73025,
+				Gid: 5000,
+				Size: 5,
+				Mtime: 1244593104,
+				Typeflag: '\x00',
+			},
+			&Header{
+				Name: "small2.txt",
+				Mode: 0444,
+				Uid: 73025,
+				Gid: 5000,
+				Size: 11,
+				Mtime: 1244593104,
+				Typeflag: '\x00',
+			},
+		},
+	},
+}
+
+var facts = map[int] string {
+	0: "1",
+	1: "1",
+	2: "2",
+	10: "3628800",
+	20: "2432902008176640000",
+	100: "933262154439441526816992388562667004907159682643816214685929" +
+		"638952175999932299156089414639761565182862536979208272237582" +
+		"51185210916864000000000000000000000000",
+}
+
+func usage() {
+	fmt.Fprintf(os.Stderr,
+		// TODO(gri): the 2nd string of this string list should not be indented
+		"usage: godoc package [name ...]\n" +
+		"	godoc -http=:6060\n")
+	flag.PrintDefaults()
+	os.Exit(2)
+}
+
+func TestReader(t *testing.T) {
+testLoop:
+	for i, test := range untarTests {
+		f, err := os.Open(test.file, os.O_RDONLY, 0444)
+		if err != nil {
+			t.Errorf("test %d: Unexpected error: %v", i, err)
+			continue
+		}
+		tr := NewReader(f)
+		for j, header := range test.headers {
+			hdr, err := tr.Next()
+			if err != nil || hdr == nil {
+				t.Errorf("test %d, entry %d: Didn't get entry: %v", i, j, err)
+				f.Close()
+				continue testLoop
+			}
+			if !reflect.DeepEqual(hdr, header) {
+				t.Errorf("test %d, entry %d: Incorrect header:\nhave %+v\nwant %+v",
+					 i, j, *hdr, *header)
+			}
+		}
+		hdr, err := tr.Next()
+		if hdr != nil || err != nil {
+			t.Errorf("test %d: Unexpected entry or error: hdr=%v err=%v", i, err)
+		}
+		f.Close()
+	}
+}
+
+// There should be exactly one linebreak after this comment.
diff --git a/libgo/go/go/printer/testdata/statements.golden b/libgo/go/go/printer/testdata/statements.golden
new file mode 100644
index 000000000..5eceb7dd5
--- /dev/null
+++ b/libgo/go/go/printer/testdata/statements.golden
@@ -0,0 +1,417 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package statements
+
+var expr bool
+
+func use(x interface{})	{}
+
+// Formatting of if-statement headers.
+func _() {
+	if {
+	}
+	if {
+	}	// no semicolon printed
+	if expr {
+	}
+	if expr {
+	}	// no semicolon printed
+	if expr {
+	}	// no parens printed
+	if expr {
+	}	// no semicolon and parens printed
+	if x := expr; {
+		use(x)
+	}
+	if x := expr; expr {
+		use(x)
+	}
+}
+
+
+// Formatting of switch-statement headers.
+func _() {
+	switch {
+	}
+	switch {
+	}	// no semicolon printed
+	switch expr {
+	}
+	switch expr {
+	}	// no semicolon printed
+	switch expr {
+	}	// no parens printed
+	switch expr {
+	}	// no semicolon and parens printed
+	switch x := expr; {
+	default:
+		use(
+			x)
+	}
+	switch x := expr; expr {
+	default:
+		use(x)
+	}
+}
+
+
+// Formatting of switch statement bodies.
+func _() {
+	switch {
+	}
+
+	switch x := 0; x {
+	case 1:
+		use(x)
+		use(x)	// followed by an empty line
+
+	case 2:	// followed by an empty line
+
+		use(x)	// followed by an empty line
+
+	case 3:	// no empty lines
+		use(x)
+		use(x)
+	}
+
+	switch x {
+	case 0:
+		use(x)
+	case 1:	// this comment should have no effect on the previous or next line
+		use(x)
+	}
+
+	switch x := 0; x {
+	case 1:
+		x = 0
+		// this comment should be indented
+	case 2:
+		x = 0
+	// this comment should not be indented, it is aligned with the next case
+	case 3:
+		x = 0
+		/* indented comment
+		   aligned
+		   aligned
+		*/
+		// bla
+		/* and more */
+	case 4:
+		x = 0
+	/* not indented comment
+	   aligned
+	   aligned
+	*/
+	// bla
+	/* and more */
+	case 5:
+	}
+}
+
+
+// Formatting of for-statement headers.
+func _() {
+	for {
+	}
+	for expr {
+	}
+	for expr {
+	}	// no parens printed
+	for {
+	}	// no semicolons printed
+	for x := expr; ; {
+		use(x)
+	}
+	for expr {
+	}	// no semicolons printed
+	for expr {
+	}	// no semicolons and parens printed
+	for ; ; expr = false {
+	}
+	for x := expr; expr; {
+		use(x)
+	}
+	for x := expr; ; expr = false {
+		use(x)
+	}
+	for ; expr; expr = false {
+	}
+	for x := expr; expr; expr = false {
+		use(x)
+	}
+	for x := range []int{} {
+		use(x)
+	}
+	for x := range []int{} {
+		use(x)
+	}	// no parens printed
+}
+
+
+// Don't remove mandatory parentheses around composite literals in control clauses.
+func _() {
+	// strip parentheses - no composite literals or composite literals don't start with a type name
+	if x {
+	}
+	if x {
+	}
+	if []T{} {
+	}
+	if []T{} {
+	}
+	if []T{} {
+	}
+
+	for x {
+	}
+	for x {
+	}
+	for []T{} {
+	}
+	for []T{} {
+	}
+	for []T{} {
+	}
+
+	switch x {
+	}
+	switch x {
+	}
+	switch []T{} {
+	}
+	switch []T{} {
+	}
+
+	for _ = range []T{T{42}} {
+	}
+
+	// leave parentheses - composite literals start with a type name
+	if (T{}) {
+	}
+	if (T{}) {
+	}
+	if (T{}) {
+	}
+
+	for (T{}) {
+	}
+	for (T{}) {
+	}
+	for (T{}) {
+	}
+
+	switch (T{}) {
+	}
+	switch (T{}) {
+	}
+
+	for _ = range (T1{T{42}}) {
+	}
+
+	if x == (T{42}[0]) {
+	}
+	if (x == T{42}[0]) {
+	}
+	if x == (T{42}[0]) {
+	}
+	if x == (T{42}[0]) {
+	}
+	if x == (T{42}[0]) {
+	}
+	if x == a+b*(T{42}[0]) {
+	}
+	if (x == a+b*T{42}[0]) {
+	}
+	if x == a+b*(T{42}[0]) {
+	}
+	if x == a+(b * (T{42}[0])) {
+	}
+	if x == a+b*(T{42}[0]) {
+	}
+	if (a + b*(T{42}[0])) == x {
+	}
+	if (a + b*(T{42}[0])) == x {
+	}
+
+	if struct{ x bool }{false}.x {
+	}
+	if (struct{ x bool }{false}.x) == false {
+	}
+	if struct{ x bool }{false}.x == false {
+	}
+}
+
+
+// Extra empty lines inside functions. Do respect source code line
+// breaks between statement boundaries but print at most one empty
+// line at a time.
+func _() {
+
+	const _ = 0
+
+	const _ = 1
+	type _ int
+	type _ float
+
+	var _ = 0
+	var x = 1
+
+	// Each use(x) call below should have at most one empty line before and after.
+	// Known bug: The first use call may have more than one empty line before
+	//            (see go/printer/nodes.go, func linebreak).
+
+
+	use(x)
+
+	if x < x {
+
+		use(x)
+
+	} else {
+
+		use(x)
+
+	}
+}
+
+
+// Formatting around labels.
+func _() {
+L:
+}
+
+
+func _() {
+	// this comment should be indented
+L:	// no semicolon needed
+}
+
+
+func _() {
+	switch 0 {
+	case 0:
+	L0:
+		;	// semicolon required
+	case 1:
+	L1:
+		;	// semicolon required
+	default:
+	L2:	// no semicolon needed
+	}
+}
+
+
+func _() {
+	f()
+L1:
+	f()
+L2:
+	;
+L3:
+}
+
+
+func _() {
+	// this comment should be indented
+L:
+}
+
+
+func _() {
+L:
+	_ = 0
+}
+
+
+func _() {
+	// this comment should be indented
+L:
+	_ = 0
+}
+
+
+func _() {
+	for {
+	L1:
+		_ = 0
+	L2:
+		_ = 0
+	}
+}
+
+
+func _() {
+	// this comment should be indented
+	for {
+	L1:
+		_ = 0
+	L2:
+		_ = 0
+	}
+}
+
+
+func _() {
+	if {
+		_ = 0
+	}
+	_ = 0	// the indentation here should not be affected by the long label name
+AnOverlongLabel:
+	_ = 0
+
+	if {
+		_ = 0
+	}
+	_ = 0
+
+L:
+	_ = 0
+}
+
+
+func _() {
+	for {
+		goto L
+	}
+L:
+
+	MoreCode()
+}
+
+
+func _() {
+	for {
+		goto L
+	}
+L:	// A comment on the same line as the label, followed by a single empty line.
+	// Known bug: There may be more than one empty line before MoreCode()
+	//            (see go/printer/nodes.go, func linebreak).
+
+
+	MoreCode()
+}
+
+
+func _() {
+	for {
+		goto L
+	}
+L:
+
+	// There should be a single empty line before this comment.
+	MoreCode()
+}
+
+
+func _() {
+	for {
+		goto AVeryLongLabelThatShouldNotAffectFormatting
+	}
+AVeryLongLabelThatShouldNotAffectFormatting:
+	// There should be a single empty line after this comment.
+
+	// There should be a single empty line before this comment.
+	MoreCode()
+}
diff --git a/libgo/go/go/printer/testdata/statements.input b/libgo/go/go/printer/testdata/statements.input
new file mode 100644
index 000000000..7819820ed
--- /dev/null
+++ b/libgo/go/go/printer/testdata/statements.input
@@ -0,0 +1,338 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package statements
+
+var expr bool
+
+func use(x interface{}) {}
+
+// Formatting of if-statement headers.
+func _() {
+	if {}
+	if;{}  // no semicolon printed
+	if expr{}
+	if;expr{}  // no semicolon printed
+	if (expr){}  // no parens printed
+	if;((expr)){}  // no semicolon and parens printed
+	if x:=expr;{
+	use(x)}
+	if x:=expr; expr {use(x)}
+}
+
+
+// Formatting of switch-statement headers.
+func _() {
+	switch {}
+	switch;{}  // no semicolon printed
+	switch expr {}
+	switch;expr{}  // no semicolon printed
+	switch (expr) {}  // no parens printed
+	switch;((expr)){}  // no semicolon and parens printed
+	switch x := expr; { default:use(
+x)
+	}
+	switch x := expr; expr {default:use(x)}
+}
+
+
+// Formatting of switch statement bodies.
+func _() {
+	switch {
+	}
+
+	switch x := 0; x {
+	case 1:
+		use(x)
+		use(x)  // followed by an empty line
+
+	case 2:  // followed by an empty line
+
+		use(x)  // followed by an empty line
+
+	case 3:  // no empty lines
+		use(x)
+		use(x)
+	}
+
+	switch x {
+	case 0:
+		use(x)
+	case 1:  // this comment should have no effect on the previous or next line
+		use(x)
+	}
+
+	switch x := 0; x {
+	case 1:
+		x = 0
+		// this comment should be indented
+	case 2:
+		x = 0
+	// this comment should not be indented, it is aligned with the next case
+	case 3:
+		x = 0
+		/* indented comment
+		   aligned
+		   aligned
+		*/
+		// bla
+		/* and more */
+	case 4:
+		x = 0
+	/* not indented comment
+	   aligned
+	   aligned
+	*/
+	// bla
+	/* and more */
+	case 5:
+	}
+}
+
+
+// Formatting of for-statement headers.
+func _() {
+	for{}
+	for expr {}
+	for (expr) {}  // no parens printed
+	for;;{}  // no semicolons printed
+	for x :=expr;; {use( x)}
+	for; expr;{}  // no semicolons printed
+	for; ((expr));{}  // no semicolons and parens printed
+	for; ; expr = false {}
+	for x :=expr; expr; {use(x)}
+	for x := expr;; expr=false {use(x)}
+	for;expr;expr =false {
+	}
+	for x := expr;expr;expr = false { use(x) }
+	for x := range []int{} { use(x) }
+	for x := range (([]int{})) { use(x) }  // no parens printed
+}
+
+
+// Don't remove mandatory parentheses around composite literals in control clauses.
+func _() {
+	// strip parentheses - no composite literals or composite literals don't start with a type name
+	if (x) {}
+	if (((x))) {}
+	if ([]T{}) {}
+	if (([]T{})) {}
+	if ; (((([]T{})))) {}
+
+	for (x) {}
+	for (((x))) {}
+	for ([]T{}) {}
+	for (([]T{})) {}
+	for ; (((([]T{})))) ; {}
+
+	switch (x) {}
+	switch (((x))) {}
+	switch ([]T{}) {}
+	switch ; (((([]T{})))) {}
+
+	for _ = range ((([]T{T{42}}))) {}
+
+	// leave parentheses - composite literals start with a type name
+	if (T{}) {}
+	if ((T{})) {}
+	if ; ((((T{})))) {}
+
+	for (T{}) {}
+	for ((T{})) {}
+	for ; ((((T{})))) ; {}
+
+	switch (T{}) {}
+	switch ; ((((T{})))) {}
+
+	for _ = range (((T1{T{42}}))) {}
+
+	if x == (T{42}[0]) {}
+	if (x == T{42}[0]) {}
+	if (x == (T{42}[0])) {}
+	if (x == (((T{42}[0])))) {}
+	if (((x == (T{42}[0])))) {}
+	if x == a + b*(T{42}[0]) {}
+	if (x == a + b*T{42}[0]) {}
+	if (x == a + b*(T{42}[0])) {}
+	if (x == a + ((b * (T{42}[0])))) {}
+	if (((x == a + b * (T{42}[0])))) {}
+	if (((a + b * (T{42}[0])) == x)) {}
+	if (((a + b * (T{42}[0])))) == x {}
+
+	if (struct{x bool}{false}.x) {}
+	if (struct{x bool}{false}.x) == false {}
+	if (struct{x bool}{false}.x == false) {}
+}
+
+
+// Extra empty lines inside functions. Do respect source code line
+// breaks between statement boundaries but print at most one empty
+// line at a time.
+func _() {
+
+	const _ = 0
+
+	const _ = 1
+	type _ int
+	type _ float
+
+	var _ = 0
+	var x = 1
+
+	// Each use(x) call below should have at most one empty line before and after.
+	// Known bug: The first use call may have more than one empty line before
+	//            (see go/printer/nodes.go, func linebreak).
+
+
+
+	use(x)
+
+	if x < x {
+
+		use(x)
+
+	} else {
+
+		use(x)
+
+	}
+}
+
+
+// Formatting around labels.
+func _() {
+	L:
+}
+
+
+func _() {
+	// this comment should be indented
+	L: ;  // no semicolon needed
+}
+
+
+func _() {
+	switch 0 {
+	case 0:
+		L0: ;  // semicolon required
+	case 1:
+		L1: ;  // semicolon required
+	default:
+		L2: ;  // no semicolon needed
+	}
+}
+
+
+func _() {
+	f()
+L1:
+	f()
+L2:
+	;
+L3:
+}
+
+
+func _() {
+	// this comment should be indented
+	L:
+}
+
+
+func _() {
+	L: _ = 0
+}
+
+
+func _() {
+	// this comment should be indented
+	L: _ = 0
+}
+
+
+func _() {
+	for {
+	L1: _ = 0
+	L2:
+		_ = 0
+	}
+}
+
+
+func _() {
+		// this comment should be indented
+	for {
+	L1: _ = 0
+	L2:
+		_ = 0
+	}
+}
+
+
+func _() {
+	if {
+		_ = 0
+	}
+	_ = 0  // the indentation here should not be affected by the long label name
+AnOverlongLabel:
+	_ = 0
+	
+	if {
+		_ = 0
+	}
+	_ = 0
+
+L:	_ = 0
+}
+
+
+func _() {
+	for {
+		goto L
+	}
+L:
+
+	MoreCode()
+}
+
+
+func _() {
+	for {
+		goto L
+	}
+L:	// A comment on the same line as the label, followed by a single empty line.
+	// Known bug: There may be more than one empty line before MoreCode()
+	//            (see go/printer/nodes.go, func linebreak).
+
+
+
+
+	MoreCode()
+}
+
+
+func _() {
+	for {
+		goto L
+	}
+L:
+
+
+
+
+	// There should be a single empty line before this comment.
+	MoreCode()
+}
+
+
+func _() {
+	for {
+		goto AVeryLongLabelThatShouldNotAffectFormatting
+	}
+AVeryLongLabelThatShouldNotAffectFormatting:
+	// There should be a single empty line after this comment.
+
+	// There should be a single empty line before this comment.
+	MoreCode()
+}
diff --git a/libgo/go/go/scanner/errors.go b/libgo/go/go/scanner/errors.go
new file mode 100644
index 000000000..47e35a710
--- /dev/null
+++ b/libgo/go/go/scanner/errors.go
@@ -0,0 +1,186 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package scanner
+
+import (
+	"container/vector"
+	"fmt"
+	"go/token"
+	"io"
+	"os"
+	"sort"
+)
+
+
+// An implementation of an ErrorHandler may be provided to the Scanner.
+// If a syntax error is encountered and a handler was installed, Error
+// is called with a position and an error message. The position points
+// to the beginning of the offending token.
+//
+type ErrorHandler interface {
+	Error(pos token.Position, msg string)
+}
+
+
+// ErrorVector implements the ErrorHandler interface. It maintains a list
+// of errors which can be retrieved with GetErrorList and GetError. The
+// zero value for an ErrorVector is an empty ErrorVector ready to use.
+//
+// A common usage pattern is to embed an ErrorVector alongside a
+// scanner in a data structure that uses the scanner. By passing a
+// reference to an ErrorVector to the scanner's Init call, default
+// error handling is obtained.
+//
+type ErrorVector struct {
+	errors vector.Vector
+}
+
+
+// Reset resets an ErrorVector to no errors.
+func (h *ErrorVector) Reset() { h.errors.Resize(0, 0) }
+
+
+// ErrorCount returns the number of errors collected.
+func (h *ErrorVector) ErrorCount() int { return h.errors.Len() }
+
+
+// Within ErrorVector, an error is represented by an Error node. The
+// position Pos, if valid, points to the beginning of the offending
+// token, and the error condition is described by Msg.
+//
+type Error struct {
+	Pos token.Position
+	Msg string
+}
+
+
+func (e *Error) String() string {
+	if e.Pos.Filename != "" || e.Pos.IsValid() {
+		// don't print ""
+		// TODO(gri) reconsider the semantics of Position.IsValid
+		return e.Pos.String() + ": " + e.Msg
+	}
+	return e.Msg
+}
+
+
+// An ErrorList is a (possibly sorted) list of Errors.
+type ErrorList []*Error
+
+
+// ErrorList implements the sort Interface.
+func (p ErrorList) Len() int      { return len(p) }
+func (p ErrorList) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
+
+
+func (p ErrorList) Less(i, j int) bool {
+	e := &p[i].Pos
+	f := &p[j].Pos
+	// Note that it is not sufficient to simply compare file offsets because
+	// the offsets do not reflect modified line information (through //line
+	// comments).
+	if e.Filename < f.Filename {
+		return true
+	}
+	if e.Filename == f.Filename {
+		if e.Line < f.Line {
+			return true
+		}
+		if e.Line == f.Line {
+			return e.Column < f.Column
+		}
+	}
+	return false
+}
+
+
+func (p ErrorList) String() string {
+	switch len(p) {
+	case 0:
+		return "unspecified error"
+	case 1:
+		return p[0].String()
+	}
+	return fmt.Sprintf("%s (and %d more errors)", p[0].String(), len(p)-1)
+}
+
+
+// These constants control the construction of the ErrorList
+// returned by GetErrors.
+//
+const (
+	Raw         = iota // leave error list unchanged
+	Sorted             // sort error list by file, line, and column number
+	NoMultiples        // sort error list and leave only the first error per line
+)
+
+
+// GetErrorList returns the list of errors collected by an ErrorVector.
+// The construction of the ErrorList returned is controlled by the mode
+// parameter. If there are no errors, the result is nil.
+//
+func (h *ErrorVector) GetErrorList(mode int) ErrorList {
+	if h.errors.Len() == 0 {
+		return nil
+	}
+
+	list := make(ErrorList, h.errors.Len())
+	for i := 0; i < h.errors.Len(); i++ {
+		list[i] = h.errors.At(i).(*Error)
+	}
+
+	if mode >= Sorted {
+		sort.Sort(list)
+	}
+
+	if mode >= NoMultiples {
+		var last token.Position // initial last.Line is != any legal error line
+		i := 0
+		for _, e := range list {
+			if e.Pos.Filename != last.Filename || e.Pos.Line != last.Line {
+				last = e.Pos
+				list[i] = e
+				i++
+			}
+		}
+		list = list[0:i]
+	}
+
+	return list
+}
+
+
+// GetError is like GetErrorList, but it returns an os.Error instead
+// so that a nil result can be assigned to an os.Error variable and
+// remains nil.
+//
+func (h *ErrorVector) GetError(mode int) os.Error {
+	if h.errors.Len() == 0 {
+		return nil
+	}
+
+	return h.GetErrorList(mode)
+}
+
+
+// ErrorVector implements the ErrorHandler interface.
+func (h *ErrorVector) Error(pos token.Position, msg string) {
+	h.errors.Push(&Error{pos, msg})
+}
+
+
+// PrintError is a utility function that prints a list of errors to w,
+// one error per line, if the err parameter is an ErrorList. Otherwise
+// it prints the err string.
+//
+func PrintError(w io.Writer, err os.Error) {
+	if list, ok := err.(ErrorList); ok {
+		for _, e := range list {
+			fmt.Fprintf(w, "%s\n", e)
+		}
+	} else {
+		fmt.Fprintf(w, "%s\n", err)
+	}
+}
diff --git a/libgo/go/go/scanner/scanner.go b/libgo/go/go/scanner/scanner.go
new file mode 100644
index 000000000..8c3205230
--- /dev/null
+++ b/libgo/go/go/scanner/scanner.go
@@ -0,0 +1,714 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// A scanner for Go source text. Takes a []byte as source which can
+// then be tokenized through repeated calls to the Scan function.
+// Typical use:
+//
+//	var s Scanner
+//	fset := token.NewFileSet()  // position information is relative to fset
+//	s.Init(fset, filename, src, nil /* no error handler */, 0)
+//	for {
+//		pos, tok, lit := s.Scan()
+//		if tok == token.EOF {
+//			break
+//		}
+//		// do something here with pos, tok, and lit
+//	}
+//
+package scanner
+
+import (
+	"bytes"
+	"go/token"
+	"path"
+	"strconv"
+	"unicode"
+	"utf8"
+)
+
+
+// A Scanner holds the scanner's internal state while processing
+// a given text.  It can be allocated as part of another data
+// structure but must be initialized via Init before use.
+//
+type Scanner struct {
+	// immutable state
+	file *token.File  // source file handle
+	dir  string       // directory portion of file.Name()
+	src  []byte       // source
+	err  ErrorHandler // error reporting; or nil
+	mode uint         // scanning mode
+
+	// scanning state
+	ch         int  // current character
+	offset     int  // character offset
+	rdOffset   int  // reading offset (position after current character)
+	lineOffset int  // current line offset
+	insertSemi bool // insert a semicolon before next newline
+
+	// public state - ok to modify
+	ErrorCount int // number of errors encountered
+}
+
+
+// Read the next Unicode char into S.ch.
+// S.ch < 0 means end-of-file.
+//
+func (S *Scanner) next() {
+	if S.rdOffset < len(S.src) {
+		S.offset = S.rdOffset
+		if S.ch == '\n' {
+			S.lineOffset = S.offset
+			S.file.AddLine(S.offset)
+		}
+		r, w := int(S.src[S.rdOffset]), 1
+		switch {
+		case r == 0:
+			S.error(S.offset, "illegal character NUL")
+		case r >= 0x80:
+			// not ASCII
+			r, w = utf8.DecodeRune(S.src[S.rdOffset:])
+			if r == utf8.RuneError && w == 1 {
+				S.error(S.offset, "illegal UTF-8 encoding")
+			}
+		}
+		S.rdOffset += w
+		S.ch = r
+	} else {
+		S.offset = len(S.src)
+		if S.ch == '\n' {
+			S.lineOffset = S.offset
+			S.file.AddLine(S.offset)
+		}
+		S.ch = -1 // eof
+	}
+}
+
+
+// The mode parameter to the Init function is a set of flags (or 0).
+// They control scanner behavior.
+//
+const (
+	ScanComments      = 1 << iota // return comments as COMMENT tokens
+	AllowIllegalChars             // do not report an error for illegal chars
+	InsertSemis                   // automatically insert semicolons
+)
+
+// Init prepares the scanner S to tokenize the text src by setting the
+// scanner at the beginning of src. The scanner uses the file set file
+// for position information and it adds line information for each line.
+// It is ok to re-use the same file when re-scanning the same file as
+// line information which is already present is ignored. Init causes a
+// panic if the file size does not match the src size.
+//
+// Calls to Scan will use the error handler err if they encounter a
+// syntax error and err is not nil. Also, for each error encountered,
+// the Scanner field ErrorCount is incremented by one. The mode parameter
+// determines how comments, illegal characters, and semicolons are handled.
+//
+// Note that Init may call err if there is an error in the first character
+// of the file.
+//
+func (S *Scanner) Init(file *token.File, src []byte, err ErrorHandler, mode uint) {
+	// Explicitly initialize all fields since a scanner may be reused.
+	if file.Size() != len(src) {
+		panic("file size does not match src len")
+	}
+	S.file = file
+	S.dir, _ = path.Split(file.Name())
+	S.src = src
+	S.err = err
+	S.mode = mode
+
+	S.ch = ' '
+	S.offset = 0
+	S.rdOffset = 0
+	S.lineOffset = 0
+	S.insertSemi = false
+	S.ErrorCount = 0
+
+	S.next()
+}
+
+
+func charString(ch int) string {
+	var s string
+	switch ch {
+	case -1:
+		return `EOF`
+	case '\a':
+		s = `\a`
+	case '\b':
+		s = `\b`
+	case '\f':
+		s = `\f`
+	case '\n':
+		s = `\n`
+	case '\r':
+		s = `\r`
+	case '\t':
+		s = `\t`
+	case '\v':
+		s = `\v`
+	case '\\':
+		s = `\\`
+	case '\'':
+		s = `\'`
+	default:
+		s = string(ch)
+	}
+	return "'" + s + "' (U+" + strconv.Itob(ch, 16) + ")"
+}
+
+
+func (S *Scanner) error(offs int, msg string) {
+	if S.err != nil {
+		S.err.Error(S.file.Position(S.file.Pos(offs)), msg)
+	}
+	S.ErrorCount++
+}
+
+
+var prefix = []byte("//line ")
+
+func (S *Scanner) interpretLineComment(text []byte) {
+	if bytes.HasPrefix(text, prefix) {
+		// get filename and line number, if any
+		if i := bytes.Index(text, []byte{':'}); i > 0 {
+			if line, err := strconv.Atoi(string(text[i+1:])); err == nil && line > 0 {
+				// valid //line filename:line comment;
+				filename := path.Clean(string(text[len(prefix):i]))
+				if filename[0] != '/' {
+					// make filename relative to current directory
+					filename = path.Join(S.dir, filename)
+				}
+				// update scanner position
+				S.file.AddLineInfo(S.lineOffset, filename, line-1) // -1 since comment applies to next line
+			}
+		}
+	}
+}
+
+
+func (S *Scanner) scanComment() {
+	// initial '/' already consumed; S.ch == '/' || S.ch == '*'
+	offs := S.offset - 1 // position of initial '/'
+
+	if S.ch == '/' {
+		//-style comment
+		S.next()
+		for S.ch != '\n' && S.ch >= 0 {
+			S.next()
+		}
+		if offs == S.lineOffset {
+			// comment starts at the beginning of the current line
+			S.interpretLineComment(S.src[offs:S.offset])
+		}
+		return
+	}
+
+	/*-style comment */
+	S.next()
+	for S.ch >= 0 {
+		ch := S.ch
+		S.next()
+		if ch == '*' && S.ch == '/' {
+			S.next()
+			return
+		}
+	}
+
+	S.error(offs, "comment not terminated")
+}
+
+
+func (S *Scanner) findLineEnd() bool {
+	// initial '/' already consumed
+
+	defer func(offs int) {
+		// reset scanner state to where it was upon calling findLineEnd
+		S.ch = '/'
+		S.offset = offs
+		S.rdOffset = offs + 1
+		S.next() // consume initial '/' again
+	}(S.offset - 1)
+
+	// read ahead until a newline, EOF, or non-comment token is found
+	for S.ch == '/' || S.ch == '*' {
+		if S.ch == '/' {
+			//-style comment always contains a newline
+			return true
+		}
+		/*-style comment: look for newline */
+		S.next()
+		for S.ch >= 0 {
+			ch := S.ch
+			if ch == '\n' {
+				return true
+			}
+			S.next()
+			if ch == '*' && S.ch == '/' {
+				S.next()
+				break
+			}
+		}
+		S.skipWhitespace() // S.insertSemi is set
+		if S.ch < 0 || S.ch == '\n' {
+			return true
+		}
+		if S.ch != '/' {
+			// non-comment token
+			return false
+		}
+		S.next() // consume '/'
+	}
+
+	return false
+}
+
+
+func isLetter(ch int) bool {
+	return 'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || ch == '_' || ch >= 0x80 && unicode.IsLetter(ch)
+}
+
+
+func isDigit(ch int) bool {
+	return '0' <= ch && ch <= '9' || ch >= 0x80 && unicode.IsDigit(ch)
+}
+
+
+func (S *Scanner) scanIdentifier() token.Token {
+	offs := S.offset
+	for isLetter(S.ch) || isDigit(S.ch) {
+		S.next()
+	}
+	return token.Lookup(S.src[offs:S.offset])
+}
+
+
+func digitVal(ch int) int {
+	switch {
+	case '0' <= ch && ch <= '9':
+		return ch - '0'
+	case 'a' <= ch && ch <= 'f':
+		return ch - 'a' + 10
+	case 'A' <= ch && ch <= 'F':
+		return ch - 'A' + 10
+	}
+	return 16 // larger than any legal digit val
+}
+
+
+func (S *Scanner) scanMantissa(base int) {
+	for digitVal(S.ch) < base {
+		S.next()
+	}
+}
+
+
+func (S *Scanner) scanNumber(seenDecimalPoint bool) token.Token {
+	// digitVal(S.ch) < 10
+	tok := token.INT
+
+	if seenDecimalPoint {
+		tok = token.FLOAT
+		S.scanMantissa(10)
+		goto exponent
+	}
+
+	if S.ch == '0' {
+		// int or float
+		offs := S.offset
+		S.next()
+		if S.ch == 'x' || S.ch == 'X' {
+			// hexadecimal int
+			S.next()
+			S.scanMantissa(16)
+		} else {
+			// octal int or float
+			seenDecimalDigit := false
+			S.scanMantissa(8)
+			if S.ch == '8' || S.ch == '9' {
+				// illegal octal int or float
+				seenDecimalDigit = true
+				S.scanMantissa(10)
+			}
+			if S.ch == '.' || S.ch == 'e' || S.ch == 'E' || S.ch == 'i' {
+				goto fraction
+			}
+			// octal int
+			if seenDecimalDigit {
+				S.error(offs, "illegal octal number")
+			}
+		}
+		goto exit
+	}
+
+	// decimal int or float
+	S.scanMantissa(10)
+
+fraction:
+	if S.ch == '.' {
+		tok = token.FLOAT
+		S.next()
+		S.scanMantissa(10)
+	}
+
+exponent:
+	if S.ch == 'e' || S.ch == 'E' {
+		tok = token.FLOAT
+		S.next()
+		if S.ch == '-' || S.ch == '+' {
+			S.next()
+		}
+		S.scanMantissa(10)
+	}
+
+	if S.ch == 'i' {
+		tok = token.IMAG
+		S.next()
+	}
+
+exit:
+	return tok
+}
+
+
+func (S *Scanner) scanEscape(quote int) {
+	offs := S.offset
+
+	var i, base, max uint32
+	switch S.ch {
+	case 'a', 'b', 'f', 'n', 'r', 't', 'v', '\\', quote:
+		S.next()
+		return
+	case '0', '1', '2', '3', '4', '5', '6', '7':
+		i, base, max = 3, 8, 255
+	case 'x':
+		S.next()
+		i, base, max = 2, 16, 255
+	case 'u':
+		S.next()
+		i, base, max = 4, 16, unicode.MaxRune
+	case 'U':
+		S.next()
+		i, base, max = 8, 16, unicode.MaxRune
+	default:
+		S.next() // always make progress
+		S.error(offs, "unknown escape sequence")
+		return
+	}
+
+	var x uint32
+	for ; i > 0 && S.ch != quote && S.ch >= 0; i-- {
+		d := uint32(digitVal(S.ch))
+		if d >= base {
+			S.error(S.offset, "illegal character in escape sequence")
+			break
+		}
+		x = x*base + d
+		S.next()
+	}
+	// in case of an error, consume remaining chars
+	for ; i > 0 && S.ch != quote && S.ch >= 0; i-- {
+		S.next()
+	}
+	if x > max || 0xd800 <= x && x < 0xe000 {
+		S.error(offs, "escape sequence is invalid Unicode code point")
+	}
+}
+
+
+func (S *Scanner) scanChar() {
+	// '\'' opening already consumed
+	offs := S.offset - 1
+
+	n := 0
+	for S.ch != '\'' {
+		ch := S.ch
+		n++
+		S.next()
+		if ch == '\n' || ch < 0 {
+			S.error(offs, "character literal not terminated")
+			n = 1
+			break
+		}
+		if ch == '\\' {
+			S.scanEscape('\'')
+		}
+	}
+
+	S.next()
+
+	if n != 1 {
+		S.error(offs, "illegal character literal")
+	}
+}
+
+
+func (S *Scanner) scanString() {
+	// '"' opening already consumed
+	offs := S.offset - 1
+
+	for S.ch != '"' {
+		ch := S.ch
+		S.next()
+		if ch == '\n' || ch < 0 {
+			S.error(offs, "string not terminated")
+			break
+		}
+		if ch == '\\' {
+			S.scanEscape('"')
+		}
+	}
+
+	S.next()
+}
+
+
+func (S *Scanner) scanRawString() {
+	// '`' opening already consumed
+	offs := S.offset - 1
+
+	for S.ch != '`' {
+		ch := S.ch
+		S.next()
+		if ch < 0 {
+			S.error(offs, "string not terminated")
+			break
+		}
+	}
+
+	S.next()
+}
+
+
+func (S *Scanner) skipWhitespace() {
+	for S.ch == ' ' || S.ch == '\t' || S.ch == '\n' && !S.insertSemi || S.ch == '\r' {
+		S.next()
+	}
+}
+
+
+// Helper functions for scanning multi-byte tokens such as >> += >>= .
+// Different routines recognize different length tok_i based on matches
+// of ch_i. If a token ends in '=', the result is tok1 or tok3
+// respectively. Otherwise, the result is tok0 if there was no other
+// matching character, or tok2 if the matching character was ch2.
+
+func (S *Scanner) switch2(tok0, tok1 token.Token) token.Token {
+	if S.ch == '=' {
+		S.next()
+		return tok1
+	}
+	return tok0
+}
+
+
+func (S *Scanner) switch3(tok0, tok1 token.Token, ch2 int, tok2 token.Token) token.Token {
+	if S.ch == '=' {
+		S.next()
+		return tok1
+	}
+	if S.ch == ch2 {
+		S.next()
+		return tok2
+	}
+	return tok0
+}
+
+
+func (S *Scanner) switch4(tok0, tok1 token.Token, ch2 int, tok2, tok3 token.Token) token.Token {
+	if S.ch == '=' {
+		S.next()
+		return tok1
+	}
+	if S.ch == ch2 {
+		S.next()
+		if S.ch == '=' {
+			S.next()
+			return tok3
+		}
+		return tok2
+	}
+	return tok0
+}
+
+
+var newline = []byte{'\n'}
+
+// Scan scans the next token and returns the token position pos,
+// the token tok, and the literal text lit corresponding to the
+// token. The source end is indicated by token.EOF.
+//
+// If the returned token is token.SEMICOLON, the corresponding
+// literal value is ";" if the semicolon was present in the source,
+// and "\n" if the semicolon was inserted because of a newline or
+// at EOF.
+//
+// For more tolerant parsing, Scan will return a valid token if
+// possible even if a syntax error was encountered. Thus, even
+// if the resulting token sequence contains no illegal tokens,
+// a client may not assume that no error occurred. Instead it
+// must check the scanner's ErrorCount or the number of calls
+// of the error handler, if there was one installed.
+//
+// Scan adds line information to the file added to the file
+// set with Init. Token positions are relative to that file
+// and thus relative to the file set.
+//
+func (S *Scanner) Scan() (token.Pos, token.Token, []byte) {
+scanAgain:
+	S.skipWhitespace()
+
+	// current token start
+	insertSemi := false
+	offs := S.offset
+	tok := token.ILLEGAL
+
+	// determine token value
+	switch ch := S.ch; {
+	case isLetter(ch):
+		tok = S.scanIdentifier()
+		switch tok {
+		case token.IDENT, token.BREAK, token.CONTINUE, token.FALLTHROUGH, token.RETURN:
+			insertSemi = true
+		}
+	case digitVal(ch) < 10:
+		insertSemi = true
+		tok = S.scanNumber(false)
+	default:
+		S.next() // always make progress
+		switch ch {
+		case -1:
+			if S.insertSemi {
+				S.insertSemi = false // EOF consumed
+				return S.file.Pos(offs), token.SEMICOLON, newline
+			}
+			tok = token.EOF
+		case '\n':
+			// we only reach here if S.insertSemi was
+			// set in the first place and exited early
+			// from S.skipWhitespace()
+			S.insertSemi = false // newline consumed
+			return S.file.Pos(offs), token.SEMICOLON, newline
+		case '"':
+			insertSemi = true
+			tok = token.STRING
+			S.scanString()
+		case '\'':
+			insertSemi = true
+			tok = token.CHAR
+			S.scanChar()
+		case '`':
+			insertSemi = true
+			tok = token.STRING
+			S.scanRawString()
+		case ':':
+			tok = S.switch2(token.COLON, token.DEFINE)
+		case '.':
+			if digitVal(S.ch) < 10 {
+				insertSemi = true
+				tok = S.scanNumber(true)
+			} else if S.ch == '.' {
+				S.next()
+				if S.ch == '.' {
+					S.next()
+					tok = token.ELLIPSIS
+				}
+			} else {
+				tok = token.PERIOD
+			}
+		case ',':
+			tok = token.COMMA
+		case ';':
+			tok = token.SEMICOLON
+		case '(':
+			tok = token.LPAREN
+		case ')':
+			insertSemi = true
+			tok = token.RPAREN
+		case '[':
+			tok = token.LBRACK
+		case ']':
+			insertSemi = true
+			tok = token.RBRACK
+		case '{':
+			tok = token.LBRACE
+		case '}':
+			insertSemi = true
+			tok = token.RBRACE
+		case '+':
+			tok = S.switch3(token.ADD, token.ADD_ASSIGN, '+', token.INC)
+			if tok == token.INC {
+				insertSemi = true
+			}
+		case '-':
+			tok = S.switch3(token.SUB, token.SUB_ASSIGN, '-', token.DEC)
+			if tok == token.DEC {
+				insertSemi = true
+			}
+		case '*':
+			tok = S.switch2(token.MUL, token.MUL_ASSIGN)
+		case '/':
+			if S.ch == '/' || S.ch == '*' {
+				// comment
+				if S.insertSemi && S.findLineEnd() {
+					// reset position to the beginning of the comment
+					S.ch = '/'
+					S.offset = offs
+					S.rdOffset = offs + 1
+					S.insertSemi = false // newline consumed
+					return S.file.Pos(offs), token.SEMICOLON, newline
+				}
+				S.scanComment()
+				if S.mode&ScanComments == 0 {
+					// skip comment
+					S.insertSemi = false // newline consumed
+					goto scanAgain
+				}
+				tok = token.COMMENT
+			} else {
+				tok = S.switch2(token.QUO, token.QUO_ASSIGN)
+			}
+		case '%':
+			tok = S.switch2(token.REM, token.REM_ASSIGN)
+		case '^':
+			tok = S.switch2(token.XOR, token.XOR_ASSIGN)
+		case '<':
+			if S.ch == '-' {
+				S.next()
+				tok = token.ARROW
+			} else {
+				tok = S.switch4(token.LSS, token.LEQ, '<', token.SHL, token.SHL_ASSIGN)
+			}
+		case '>':
+			tok = S.switch4(token.GTR, token.GEQ, '>', token.SHR, token.SHR_ASSIGN)
+		case '=':
+			tok = S.switch2(token.ASSIGN, token.EQL)
+		case '!':
+			tok = S.switch2(token.NOT, token.NEQ)
+		case '&':
+			if S.ch == '^' {
+				S.next()
+				tok = S.switch2(token.AND_NOT, token.AND_NOT_ASSIGN)
+			} else {
+				tok = S.switch3(token.AND, token.AND_ASSIGN, '&', token.LAND)
+			}
+		case '|':
+			tok = S.switch3(token.OR, token.OR_ASSIGN, '|', token.LOR)
+		default:
+			if S.mode&AllowIllegalChars == 0 {
+				S.error(offs, "illegal character "+charString(ch))
+			}
+			insertSemi = S.insertSemi // preserve insertSemi info
+		}
+	}
+
+	if S.mode&InsertSemis != 0 {
+		S.insertSemi = insertSemi
+	}
+	return S.file.Pos(offs), tok, S.src[offs:S.offset]
+}
diff --git a/libgo/go/go/scanner/scanner_test.go b/libgo/go/go/scanner/scanner_test.go
new file mode 100644
index 000000000..1c3b6728c
--- /dev/null
+++ b/libgo/go/go/scanner/scanner_test.go
@@ -0,0 +1,672 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package scanner
+
+import (
+	"go/token"
+	"os"
+	"testing"
+)
+
+
+var fset = token.NewFileSet()
+
+
+const /* class */ (
+	special = iota
+	literal
+	operator
+	keyword
+)
+
+
+func tokenclass(tok token.Token) int {
+	switch {
+	case tok.IsLiteral():
+		return literal
+	case tok.IsOperator():
+		return operator
+	case tok.IsKeyword():
+		return keyword
+	}
+	return special
+}
+
+
+type elt struct {
+	tok   token.Token
+	lit   string
+	class int
+}
+
+
+var tokens = [...]elt{
+	// Special tokens
+	{token.COMMENT, "/* a comment */", special},
+	{token.COMMENT, "// a comment \n", special},
+
+	// Identifiers and basic type literals
+	{token.IDENT, "foobar", literal},
+	{token.IDENT, "a۰۱۸", literal},
+	{token.IDENT, "foo६४", literal},
+	{token.IDENT, "bar９８７６", literal},
+	{token.INT, "0", literal},
+	{token.INT, "1", literal},
+	{token.INT, "123456789012345678890", literal},
+	{token.INT, "01234567", literal},
+	{token.INT, "0xcafebabe", literal},
+	{token.FLOAT, "0.", literal},
+	{token.FLOAT, ".0", literal},
+	{token.FLOAT, "3.14159265", literal},
+	{token.FLOAT, "1e0", literal},
+	{token.FLOAT, "1e+100", literal},
+	{token.FLOAT, "1e-100", literal},
+	{token.FLOAT, "2.71828e-1000", literal},
+	{token.IMAG, "0i", literal},
+	{token.IMAG, "1i", literal},
+	{token.IMAG, "012345678901234567889i", literal},
+	{token.IMAG, "123456789012345678890i", literal},
+	{token.IMAG, "0.i", literal},
+	{token.IMAG, ".0i", literal},
+	{token.IMAG, "3.14159265i", literal},
+	{token.IMAG, "1e0i", literal},
+	{token.IMAG, "1e+100i", literal},
+	{token.IMAG, "1e-100i", literal},
+	{token.IMAG, "2.71828e-1000i", literal},
+	{token.CHAR, "'a'", literal},
+	{token.CHAR, "'\\000'", literal},
+	{token.CHAR, "'\\xFF'", literal},
+	{token.CHAR, "'\\uff16'", literal},
+	{token.CHAR, "'\\U0000ff16'", literal},
+	{token.STRING, "`foobar`", literal},
+	{token.STRING, "`" + `foo
+	                        bar` +
+		"`",
+		literal,
+	},
+
+	// Operators and delimitors
+	{token.ADD, "+", operator},
+	{token.SUB, "-", operator},
+	{token.MUL, "*", operator},
+	{token.QUO, "/", operator},
+	{token.REM, "%", operator},
+
+	{token.AND, "&", operator},
+	{token.OR, "|", operator},
+	{token.XOR, "^", operator},
+	{token.SHL, "<<", operator},
+	{token.SHR, ">>", operator},
+	{token.AND_NOT, "&^", operator},
+
+	{token.ADD_ASSIGN, "+=", operator},
+	{token.SUB_ASSIGN, "-=", operator},
+	{token.MUL_ASSIGN, "*=", operator},
+	{token.QUO_ASSIGN, "/=", operator},
+	{token.REM_ASSIGN, "%=", operator},
+
+	{token.AND_ASSIGN, "&=", operator},
+	{token.OR_ASSIGN, "|=", operator},
+	{token.XOR_ASSIGN, "^=", operator},
+	{token.SHL_ASSIGN, "<<=", operator},
+	{token.SHR_ASSIGN, ">>=", operator},
+	{token.AND_NOT_ASSIGN, "&^=", operator},
+
+	{token.LAND, "&&", operator},
+	{token.LOR, "||", operator},
+	{token.ARROW, "<-", operator},
+	{token.INC, "++", operator},
+	{token.DEC, "--", operator},
+
+	{token.EQL, "==", operator},
+	{token.LSS, "<", operator},
+	{token.GTR, ">", operator},
+	{token.ASSIGN, "=", operator},
+	{token.NOT, "!", operator},
+
+	{token.NEQ, "!=", operator},
+	{token.LEQ, "<=", operator},
+	{token.GEQ, ">=", operator},
+	{token.DEFINE, ":=", operator},
+	{token.ELLIPSIS, "...", operator},
+
+	{token.LPAREN, "(", operator},
+	{token.LBRACK, "[", operator},
+	{token.LBRACE, "{", operator},
+	{token.COMMA, ",", operator},
+	{token.PERIOD, ".", operator},
+
+	{token.RPAREN, ")", operator},
+	{token.RBRACK, "]", operator},
+	{token.RBRACE, "}", operator},
+	{token.SEMICOLON, ";", operator},
+	{token.COLON, ":", operator},
+
+	// Keywords
+	{token.BREAK, "break", keyword},
+	{token.CASE, "case", keyword},
+	{token.CHAN, "chan", keyword},
+	{token.CONST, "const", keyword},
+	{token.CONTINUE, "continue", keyword},
+
+	{token.DEFAULT, "default", keyword},
+	{token.DEFER, "defer", keyword},
+	{token.ELSE, "else", keyword},
+	{token.FALLTHROUGH, "fallthrough", keyword},
+	{token.FOR, "for", keyword},
+
+	{token.FUNC, "func", keyword},
+	{token.GO, "go", keyword},
+	{token.GOTO, "goto", keyword},
+	{token.IF, "if", keyword},
+	{token.IMPORT, "import", keyword},
+
+	{token.INTERFACE, "interface", keyword},
+	{token.MAP, "map", keyword},
+	{token.PACKAGE, "package", keyword},
+	{token.RANGE, "range", keyword},
+	{token.RETURN, "return", keyword},
+
+	{token.SELECT, "select", keyword},
+	{token.STRUCT, "struct", keyword},
+	{token.SWITCH, "switch", keyword},
+	{token.TYPE, "type", keyword},
+	{token.VAR, "var", keyword},
+}
+
+
+const whitespace = "  \t  \n\n\n" // to separate tokens
+
+type testErrorHandler struct {
+	t *testing.T
+}
+
+func (h *testErrorHandler) Error(pos token.Position, msg string) {
+	h.t.Errorf("Error() called (msg = %s)", msg)
+}
+
+
+func newlineCount(s string) int {
+	n := 0
+	for i := 0; i < len(s); i++ {
+		if s[i] == '\n' {
+			n++
+		}
+	}
+	return n
+}
+
+
+func checkPos(t *testing.T, lit string, p token.Pos, expected token.Position) {
+	pos := fset.Position(p)
+	if pos.Filename != expected.Filename {
+		t.Errorf("bad filename for %q: got %s, expected %s", lit, pos.Filename, expected.Filename)
+	}
+	if pos.Offset != expected.Offset {
+		t.Errorf("bad position for %q: got %d, expected %d", lit, pos.Offset, expected.Offset)
+	}
+	if pos.Line != expected.Line {
+		t.Errorf("bad line for %q: got %d, expected %d", lit, pos.Line, expected.Line)
+	}
+	if pos.Column != expected.Column {
+		t.Errorf("bad column for %q: got %d, expected %d", lit, pos.Column, expected.Column)
+	}
+}
+
+
+// Verify that calling Scan() provides the correct results.
+func TestScan(t *testing.T) {
+	// make source
+	var src string
+	for _, e := range tokens {
+		src += e.lit + whitespace
+	}
+	src_linecount := newlineCount(src) + 1
+	whitespace_linecount := newlineCount(whitespace)
+
+	// verify scan
+	var s Scanner
+	s.Init(fset.AddFile("", fset.Base(), len(src)), []byte(src), &testErrorHandler{t}, ScanComments)
+	index := 0
+	epos := token.Position{"", 0, 1, 1} // expected position
+	for {
+		pos, tok, litb := s.Scan()
+		e := elt{token.EOF, "", special}
+		if index < len(tokens) {
+			e = tokens[index]
+		}
+		lit := string(litb)
+		if tok == token.EOF {
+			lit = ""
+			epos.Line = src_linecount
+			epos.Column = 1
+		}
+		checkPos(t, lit, pos, epos)
+		if tok != e.tok {
+			t.Errorf("bad token for %q: got %s, expected %s", lit, tok.String(), e.tok.String())
+		}
+		if e.tok.IsLiteral() && lit != e.lit {
+			t.Errorf("bad literal for %q: got %q, expected %q", lit, lit, e.lit)
+		}
+		if tokenclass(tok) != e.class {
+			t.Errorf("bad class for %q: got %d, expected %d", lit, tokenclass(tok), e.class)
+		}
+		epos.Offset += len(lit) + len(whitespace)
+		epos.Line += newlineCount(lit) + whitespace_linecount
+		if tok == token.COMMENT && litb[1] == '/' {
+			// correct for unaccounted '/n' in //-style comment
+			epos.Offset++
+			epos.Line++
+		}
+		index++
+		if tok == token.EOF {
+			break
+		}
+	}
+	if s.ErrorCount != 0 {
+		t.Errorf("found %d errors", s.ErrorCount)
+	}
+}
+
+
+func checkSemi(t *testing.T, line string, mode uint) {
+	var S Scanner
+	file := fset.AddFile("TestSemis", fset.Base(), len(line))
+	S.Init(file, []byte(line), nil, mode)
+	pos, tok, lit := S.Scan()
+	for tok != token.EOF {
+		if tok == token.ILLEGAL {
+			// the illegal token literal indicates what
+			// kind of semicolon literal to expect
+			semiLit := "\n"
+			if lit[0] == '#' {
+				semiLit = ";"
+			}
+			// next token must be a semicolon
+			semiPos := file.Position(pos)
+			semiPos.Offset++
+			semiPos.Column++
+			pos, tok, lit = S.Scan()
+			if tok == token.SEMICOLON {
+				if string(lit) != semiLit {
+					t.Errorf(`bad literal for %q: got %q, expected %q`, line, lit, semiLit)
+				}
+				checkPos(t, line, pos, semiPos)
+			} else {
+				t.Errorf("bad token for %q: got %s, expected ;", line, tok.String())
+			}
+		} else if tok == token.SEMICOLON {
+			t.Errorf("bad token for %q: got ;, expected no ;", line)
+		}
+		pos, tok, lit = S.Scan()
+	}
+}
+
+
+var lines = []string{
+	// # indicates a semicolon present in the source
+	// $ indicates an automatically inserted semicolon
+	"",
+	"#;",
+	"foo$\n",
+	"123$\n",
+	"1.2$\n",
+	"'x'$\n",
+	`"x"` + "$\n",
+	"`x`$\n",
+
+	"+\n",
+	"-\n",
+	"*\n",
+	"/\n",
+	"%\n",
+
+	"&\n",
+	"|\n",
+	"^\n",
+	"<<\n",
+	">>\n",
+	"&^\n",
+
+	"+=\n",
+	"-=\n",
+	"*=\n",
+	"/=\n",
+	"%=\n",
+
+	"&=\n",
+	"|=\n",
+	"^=\n",
+	"<<=\n",
+	">>=\n",
+	"&^=\n",
+
+	"&&\n",
+	"||\n",
+	"<-\n",
+	"++$\n",
+	"--$\n",
+
+	"==\n",
+	"<\n",
+	">\n",
+	"=\n",
+	"!\n",
+
+	"!=\n",
+	"<=\n",
+	">=\n",
+	":=\n",
+	"...\n",
+
+	"(\n",
+	"[\n",
+	"{\n",
+	",\n",
+	".\n",
+
+	")$\n",
+	"]$\n",
+	"}$\n",
+	"#;\n",
+	":\n",
+
+	"break$\n",
+	"case\n",
+	"chan\n",
+	"const\n",
+	"continue$\n",
+
+	"default\n",
+	"defer\n",
+	"else\n",
+	"fallthrough$\n",
+	"for\n",
+
+	"func\n",
+	"go\n",
+	"goto\n",
+	"if\n",
+	"import\n",
+
+	"interface\n",
+	"map\n",
+	"package\n",
+	"range\n",
+	"return$\n",
+
+	"select\n",
+	"struct\n",
+	"switch\n",
+	"type\n",
+	"var\n",
+
+	"foo$//comment\n",
+	"foo$//comment",
+	"foo$/*comment*/\n",
+	"foo$/*\n*/",
+	"foo$/*comment*/    \n",
+	"foo$/*\n*/    ",
+
+	"foo    $// comment\n",
+	"foo    $// comment",
+	"foo    $/*comment*/\n",
+	"foo    $/*\n*/",
+	"foo    $/*  */ /* \n */ bar$/**/\n",
+	"foo    $/*0*/ /*1*/ /*2*/\n",
+
+	"foo    $/*comment*/    \n",
+	"foo    $/*0*/ /*1*/ /*2*/    \n",
+	"foo	$/**/ /*-------------*/       /*----\n*/bar       $/*  \n*/baa$\n",
+	"foo    $/* an EOF terminates a line */",
+	"foo    $/* an EOF terminates a line */ /*",
+	"foo    $/* an EOF terminates a line */ //",
+
+	"package main$\n\nfunc main() {\n\tif {\n\t\treturn /* */ }$\n}$\n",
+	"package main$",
+}
+
+
+func TestSemis(t *testing.T) {
+	for _, line := range lines {
+		checkSemi(t, line, AllowIllegalChars|InsertSemis)
+		checkSemi(t, line, AllowIllegalChars|InsertSemis|ScanComments)
+
+		// if the input ended in newlines, the input must tokenize the
+		// same with or without those newlines
+		for i := len(line) - 1; i >= 0 && line[i] == '\n'; i-- {
+			checkSemi(t, line[0:i], AllowIllegalChars|InsertSemis)
+			checkSemi(t, line[0:i], AllowIllegalChars|InsertSemis|ScanComments)
+		}
+	}
+}
+
+
+var segments = []struct {
+	srcline  string // a line of source text
+	filename string // filename for current token
+	line     int    // line number for current token
+}{
+	// exactly one token per line since the test consumes one token per segment
+	{"  line1", "dir/TestLineComments", 1},
+	{"\nline2", "dir/TestLineComments", 2},
+	{"\nline3  //line File1.go:100", "dir/TestLineComments", 3}, // bad line comment, ignored
+	{"\nline4", "dir/TestLineComments", 4},
+	{"\n//line File1.go:100\n  line100", "dir/File1.go", 100},
+	{"\n//line File2.go:200\n  line200", "dir/File2.go", 200},
+	{"\n//line :1\n  line1", "dir", 1},
+	{"\n//line foo:42\n  line42", "dir/foo", 42},
+	{"\n //line foo:42\n  line44", "dir/foo", 44},           // bad line comment, ignored
+	{"\n//line foo 42\n  line46", "dir/foo", 46},            // bad line comment, ignored
+	{"\n//line foo:42 extra text\n  line48", "dir/foo", 48}, // bad line comment, ignored
+	{"\n//line /bar:42\n  line42", "/bar", 42},
+	{"\n//line ./foo:42\n  line42", "dir/foo", 42},
+	{"\n//line a/b/c/File1.go:100\n  line100", "dir/a/b/c/File1.go", 100},
+}
+
+
+// Verify that comments of the form "//line filename:line" are interpreted correctly.
+func TestLineComments(t *testing.T) {
+	// make source
+	var src string
+	for _, e := range segments {
+		src += e.srcline
+	}
+
+	// verify scan
+	var S Scanner
+	file := fset.AddFile("dir/TestLineComments", fset.Base(), len(src))
+	S.Init(file, []byte(src), nil, 0)
+	for _, s := range segments {
+		p, _, lit := S.Scan()
+		pos := file.Position(p)
+		checkPos(t, string(lit), p, token.Position{s.filename, pos.Offset, s.line, pos.Column})
+	}
+
+	if S.ErrorCount != 0 {
+		t.Errorf("found %d errors", S.ErrorCount)
+	}
+}
+
+
+// Verify that initializing the same scanner more then once works correctly.
+func TestInit(t *testing.T) {
+	var s Scanner
+
+	// 1st init
+	src1 := "if true { }"
+	f1 := fset.AddFile("src1", fset.Base(), len(src1))
+	s.Init(f1, []byte(src1), nil, 0)
+	if f1.Size() != len(src1) {
+		t.Errorf("bad file size: got %d, expected %d", f1.Size(), len(src1))
+	}
+	s.Scan()              // if
+	s.Scan()              // true
+	_, tok, _ := s.Scan() // {
+	if tok != token.LBRACE {
+		t.Errorf("bad token: got %s, expected %s", tok.String(), token.LBRACE)
+	}
+
+	// 2nd init
+	src2 := "go true { ]"
+	f2 := fset.AddFile("src2", fset.Base(), len(src2))
+	s.Init(f2, []byte(src2), nil, 0)
+	if f2.Size() != len(src2) {
+		t.Errorf("bad file size: got %d, expected %d", f2.Size(), len(src2))
+	}
+	_, tok, _ = s.Scan() // go
+	if tok != token.GO {
+		t.Errorf("bad token: got %s, expected %s", tok.String(), token.GO)
+	}
+
+	if s.ErrorCount != 0 {
+		t.Errorf("found %d errors", s.ErrorCount)
+	}
+}
+
+
+func TestIllegalChars(t *testing.T) {
+	var s Scanner
+
+	const src = "*?*$*@*"
+	file := fset.AddFile("", fset.Base(), len(src))
+	s.Init(file, []byte(src), &testErrorHandler{t}, AllowIllegalChars)
+	for offs, ch := range src {
+		pos, tok, lit := s.Scan()
+		if poffs := file.Offset(pos); poffs != offs {
+			t.Errorf("bad position for %s: got %d, expected %d", string(lit), poffs, offs)
+		}
+		if tok == token.ILLEGAL && string(lit) != string(ch) {
+			t.Errorf("bad token: got %s, expected %s", string(lit), string(ch))
+		}
+	}
+
+	if s.ErrorCount != 0 {
+		t.Errorf("found %d errors", s.ErrorCount)
+	}
+}
+
+
+func TestStdErrorHander(t *testing.T) {
+	const src = "@\n" + // illegal character, cause an error
+		"@ @\n" + // two errors on the same line
+		"//line File2:20\n" +
+		"@\n" + // different file, but same line
+		"//line File2:1\n" +
+		"@ @\n" + // same file, decreasing line number
+		"//line File1:1\n" +
+		"@ @ @" // original file, line 1 again
+
+	v := new(ErrorVector)
+	var s Scanner
+	s.Init(fset.AddFile("File1", fset.Base(), len(src)), []byte(src), v, 0)
+	for {
+		if _, tok, _ := s.Scan(); tok == token.EOF {
+			break
+		}
+	}
+
+	list := v.GetErrorList(Raw)
+	if len(list) != 9 {
+		t.Errorf("found %d raw errors, expected 9", len(list))
+		PrintError(os.Stderr, list)
+	}
+
+	list = v.GetErrorList(Sorted)
+	if len(list) != 9 {
+		t.Errorf("found %d sorted errors, expected 9", len(list))
+		PrintError(os.Stderr, list)
+	}
+
+	list = v.GetErrorList(NoMultiples)
+	if len(list) != 4 {
+		t.Errorf("found %d one-per-line errors, expected 4", len(list))
+		PrintError(os.Stderr, list)
+	}
+
+	if v.ErrorCount() != s.ErrorCount {
+		t.Errorf("found %d errors, expected %d", v.ErrorCount(), s.ErrorCount)
+	}
+}
+
+
+type errorCollector struct {
+	cnt int            // number of errors encountered
+	msg string         // last error message encountered
+	pos token.Position // last error position encountered
+}
+
+
+func (h *errorCollector) Error(pos token.Position, msg string) {
+	h.cnt++
+	h.msg = msg
+	h.pos = pos
+}
+
+
+func checkError(t *testing.T, src string, tok token.Token, pos int, err string) {
+	var s Scanner
+	var h errorCollector
+	s.Init(fset.AddFile("", fset.Base(), len(src)), []byte(src), &h, ScanComments)
+	_, tok0, _ := s.Scan()
+	_, tok1, _ := s.Scan()
+	if tok0 != tok {
+		t.Errorf("%q: got %s, expected %s", src, tok0, tok)
+	}
+	if tok1 != token.EOF {
+		t.Errorf("%q: got %s, expected EOF", src, tok1)
+	}
+	cnt := 0
+	if err != "" {
+		cnt = 1
+	}
+	if h.cnt != cnt {
+		t.Errorf("%q: got cnt %d, expected %d", src, h.cnt, cnt)
+	}
+	if h.msg != err {
+		t.Errorf("%q: got msg %q, expected %q", src, h.msg, err)
+	}
+	if h.pos.Offset != pos {
+		t.Errorf("%q: got offset %d, expected %d", src, h.pos.Offset, pos)
+	}
+}
+
+
+var errors = []struct {
+	src string
+	tok token.Token
+	pos int
+	err string
+}{
+	{`#`, token.ILLEGAL, 0, "illegal character '#' (U+23)"},
+	{`' '`, token.CHAR, 0, ""},
+	{`''`, token.CHAR, 0, "illegal character literal"},
+	{`'\8'`, token.CHAR, 2, "unknown escape sequence"},
+	{`'\08'`, token.CHAR, 3, "illegal character in escape sequence"},
+	{`'\x0g'`, token.CHAR, 4, "illegal character in escape sequence"},
+	{`'\Uffffffff'`, token.CHAR, 2, "escape sequence is invalid Unicode code point"},
+	{`'`, token.CHAR, 0, "character literal not terminated"},
+	{`""`, token.STRING, 0, ""},
+	{`"`, token.STRING, 0, "string not terminated"},
+	{"``", token.STRING, 0, ""},
+	{"`", token.STRING, 0, "string not terminated"},
+	{"/**/", token.COMMENT, 0, ""},
+	{"/*", token.COMMENT, 0, "comment not terminated"},
+	{"077", token.INT, 0, ""},
+	{"078.", token.FLOAT, 0, ""},
+	{"07801234567.", token.FLOAT, 0, ""},
+	{"078e0", token.FLOAT, 0, ""},
+	{"078", token.INT, 0, "illegal octal number"},
+	{"07800000009", token.INT, 0, "illegal octal number"},
+	{"\"abc\x00def\"", token.STRING, 4, "illegal character NUL"},
+	{"\"abc\x80def\"", token.STRING, 4, "illegal UTF-8 encoding"},
+}
+
+
+func TestScanErrors(t *testing.T) {
+	for _, e := range errors {
+		checkError(t, e.src, e.tok, e.pos, e.err)
+	}
+}
diff --git a/libgo/go/go/token/position.go b/libgo/go/go/token/position.go
new file mode 100644
index 000000000..0044a0ed7
--- /dev/null
+++ b/libgo/go/go/token/position.go
@@ -0,0 +1,409 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// TODO(gri) consider making this a separate package outside the go directory.
+
+package token
+
+import (
+	"fmt"
+	"sort"
+	"sync"
+)
+
+
+// Position describes an arbitrary source position
+// including the file, line, and column location.
+// A Position is valid if the line number is > 0.
+//
+type Position struct {
+	Filename string // filename, if any
+	Offset   int    // offset, starting at 0
+	Line     int    // line number, starting at 1
+	Column   int    // column number, starting at 1 (character count)
+}
+
+
+// IsValid returns true if the position is valid.
+func (pos *Position) IsValid() bool { return pos.Line > 0 }
+
+
+// String returns a string in one of several forms:
+//
+//	file:line:column    valid position with file name
+//	line:column         valid position without file name
+//	file                invalid position with file name
+//	-                   invalid position without file name
+//
+func (pos Position) String() string {
+	s := pos.Filename
+	if pos.IsValid() {
+		if s != "" {
+			s += ":"
+		}
+		s += fmt.Sprintf("%d:%d", pos.Line, pos.Column)
+	}
+	if s == "" {
+		s = "-"
+	}
+	return s
+}
+
+
+// Pos is a compact encoding of a source position within a file set.
+// It can be converted into a Position for a more convenient, but much
+// larger, representation.
+//
+// The Pos value for a given file is a number in the range [base, base+size],
+// where base and size are specified when adding the file to the file set via
+// AddFile.
+//
+// To create the Pos value for a specific source offset, first add
+// the respective file to the current file set (via FileSet.AddFile)
+// and then call File.Pos(offset) for that file. Given a Pos value p
+// for a specific file set fset, the corresponding Position value is
+// obtained by calling fset.Position(p).
+//
+// Pos values can be compared directly with the usual comparison operators:
+// If two Pos values p and q are in the same file, comparing p and q is
+// equivalent to comparing the respective source file offsets. If p and q
+// are in different files, p < q is true if the file implied by p was added
+// to the respective file set before the file implied by q.
+//
+type Pos int
+
+
+// The zero value for Pos is NoPos; there is no file and line information
+// associated with it, and NoPos().IsValid() is false. NoPos is always
+// smaller than any other Pos value. The corresponding Position value
+// for NoPos is the zero value for Position.
+// 
+const NoPos Pos = 0
+
+
+// IsValid returns true if the position is valid.
+func (p Pos) IsValid() bool {
+	return p != NoPos
+}
+
+
+func searchFiles(a []*File, x int) int {
+	return sort.Search(len(a), func(i int) bool { return a[i].base > x }) - 1
+}
+
+
+func (s *FileSet) file(p Pos) *File {
+	if i := searchFiles(s.files, int(p)); i >= 0 {
+		f := s.files[i]
+		// f.base <= int(p) by definition of searchFiles
+		if int(p) <= f.base+f.size {
+			return f
+		}
+	}
+	return nil
+}
+
+
+// File returns the file which contains the position p.
+// If no such file is found (for instance for p == NoPos),
+// the result is nil.
+//
+func (s *FileSet) File(p Pos) (f *File) {
+	if p != NoPos {
+		s.mutex.RLock()
+		f = s.file(p)
+		s.mutex.RUnlock()
+	}
+	return
+}
+
+
+func (f *File) position(p Pos) (pos Position) {
+	offset := int(p) - f.base
+	pos.Offset = offset
+	pos.Filename, pos.Line, pos.Column = f.info(offset)
+	return
+}
+
+
+// Position converts a Pos in the fileset into a general Position.
+func (s *FileSet) Position(p Pos) (pos Position) {
+	if p != NoPos {
+		// TODO(gri) consider optimizing the case where p
+		//           is in the last file addded, or perhaps
+		//           looked at - will eliminate one level
+		//           of search
+		s.mutex.RLock()
+		if f := s.file(p); f != nil {
+			pos = f.position(p)
+		}
+		s.mutex.RUnlock()
+	}
+	return
+}
+
+
+type lineInfo struct {
+	offset   int
+	filename string
+	line     int
+}
+
+
+// AddLineInfo adds alternative file and line number information for
+// a given file offset. The offset must be larger than the offset for
+// the previously added alternative line info and not larger than the
+// file size; otherwise the information is ignored.
+//
+// AddLineInfo is typically used to register alternative position
+// information for //line filename:line comments in source files.
+//
+func (f *File) AddLineInfo(offset int, filename string, line int) {
+	f.set.mutex.Lock()
+	if i := len(f.infos); i == 0 || f.infos[i-1].offset < offset && offset <= f.size {
+		f.infos = append(f.infos, lineInfo{offset, filename, line})
+	}
+	f.set.mutex.Unlock()
+}
+
+
+// A File is a handle for a file belonging to a FileSet.
+// A File has a name, size, and line offset table.
+//
+type File struct {
+	set  *FileSet
+	name string // file name as provided to AddFile
+	base int    // Pos value range for this file is [base...base+size]
+	size int    // file size as provided to AddFile
+
+	// lines and infos are protected by set.mutex
+	lines []int
+	infos []lineInfo
+}
+
+
+// Name returns the file name of file f as registered with AddFile.
+func (f *File) Name() string {
+	return f.name
+}
+
+
+// Base returns the base offset of file f as registered with AddFile.
+func (f *File) Base() int {
+	return f.base
+}
+
+
+// Size returns the size of file f as registered with AddFile.
+func (f *File) Size() int {
+	return f.size
+}
+
+
+// LineCount returns the number of lines in file f.
+func (f *File) LineCount() int {
+	f.set.mutex.RLock()
+	n := len(f.lines)
+	f.set.mutex.RUnlock()
+	return n
+}
+
+
+// AddLine adds the line offset for a new line.
+// The line offset must be larger than the offset for the previous line
+// and not larger than the file size; otherwise the line offset is ignored.
+//
+func (f *File) AddLine(offset int) {
+	f.set.mutex.Lock()
+	if i := len(f.lines); (i == 0 || f.lines[i-1] < offset) && offset <= f.size {
+		f.lines = append(f.lines, offset)
+	}
+	f.set.mutex.Unlock()
+}
+
+
+// SetLines sets all line offsets for a file and returns true if successful.
+// Each line offset must be larger than the offset for the previous line
+// and not larger than the file size; otherwise the SetLines fails and returns
+// false.
+//
+func (f *File) SetLines(lines []int) bool {
+	// verify validity of lines table
+	size := f.size
+	for i, offset := range lines {
+		if i > 0 && offset <= lines[i-1] || size < offset {
+			return false
+		}
+	}
+
+	// set lines table
+	f.set.mutex.Lock()
+	f.lines = lines
+	f.set.mutex.Unlock()
+	return true
+}
+
+
+// Pos returns the Pos value for the given file offset;
+// the offset must be <= f.Size().
+// f.Pos(f.Offset(p)) == p.
+//
+func (f *File) Pos(offset int) Pos {
+	if offset > f.size {
+		panic("illegal file offset")
+	}
+	return Pos(f.base + offset)
+}
+
+
+// Offset returns the offset for the given file position p;
+// p must be a valid Pos value in that file.
+// f.Offset(f.Pos(offset)) == offset.
+//
+func (f *File) Offset(p Pos) int {
+	if int(p) < f.base || int(p) > f.base+f.size {
+		panic("illegal Pos value")
+	}
+	return int(p) - f.base
+}
+
+
+// Line returns the line number for the given file position p;
+// p must be a Pos value in that file or NoPos.
+//
+func (f *File) Line(p Pos) int {
+	// TODO(gri) this can be implemented much more efficiently
+	return f.Position(p).Line
+}
+
+
+// Position returns the Position value for the given file position p;
+// p must be a Pos value in that file or NoPos.
+//
+func (f *File) Position(p Pos) (pos Position) {
+	if p != NoPos {
+		if int(p) < f.base || int(p) > f.base+f.size {
+			panic("illegal Pos value")
+		}
+		pos = f.position(p)
+	}
+	return
+}
+
+
+func searchUints(a []int, x int) int {
+	return sort.Search(len(a), func(i int) bool { return a[i] > x }) - 1
+}
+
+
+func searchLineInfos(a []lineInfo, x int) int {
+	return sort.Search(len(a), func(i int) bool { return a[i].offset > x }) - 1
+}
+
+
+// info returns the file name, line, and column number for a file offset.
+func (f *File) info(offset int) (filename string, line, column int) {
+	filename = f.name
+	if i := searchUints(f.lines, offset); i >= 0 {
+		line, column = i+1, offset-f.lines[i]+1
+	}
+	if i := searchLineInfos(f.infos, offset); i >= 0 {
+		alt := &f.infos[i]
+		filename = alt.filename
+		if i := searchUints(f.lines, alt.offset); i >= 0 {
+			line += alt.line - i - 1
+		}
+	}
+	return
+}
+
+
+// A FileSet represents a set of source files.
+// Methods of file sets are synchronized; multiple goroutines
+// may invoke them concurrently.
+//
+type FileSet struct {
+	mutex sync.RWMutex  // protects the file set
+	base  int           // base offset for the next file
+	files []*File       // list of files in the order added to the set
+	index map[*File]int // file -> files index for quick lookup
+}
+
+
+// NewFileSet creates a new file set.
+func NewFileSet() *FileSet {
+	s := new(FileSet)
+	s.base = 1 // 0 == NoPos
+	s.index = make(map[*File]int)
+	return s
+}
+
+
+// Base returns the minimum base offset that must be provided to
+// AddFile when adding the next file.
+//
+func (s *FileSet) Base() int {
+	s.mutex.RLock()
+	b := s.base
+	s.mutex.RUnlock()
+	return b
+
+}
+
+
+// AddFile adds a new file with a given filename, base offset, and file size
+// to the file set s and returns the file. Multiple files may have the same
+// name. The base offset must not be smaller than the FileSet's Base(), and
+// size must not be negative.
+//
+// Adding the file will set the file set's Base() value to base + size + 1
+// as the minimum base value for the next file. The following relationship
+// exists between a Pos value p for a given file offset offs:
+//
+//	int(p) = base + offs
+//
+// with offs in the range [0, size] and thus p in the range [base, base+size].
+// For convenience, File.Pos may be used to create file-specific position
+// values from a file offset.
+//
+func (s *FileSet) AddFile(filename string, base, size int) *File {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	if base < s.base || size < 0 {
+		panic("illegal base or size")
+	}
+	// base >= s.base && size >= 0
+	f := &File{s, filename, base, size, []int{0}, nil}
+	base += size + 1 // +1 because EOF also has a position
+	if base < 0 {
+		panic("token.Pos offset overflow (> 2G of source code in file set)")
+	}
+	// add the file to the file set
+	s.base = base
+	s.index[f] = len(s.files)
+	s.files = append(s.files, f)
+	return f
+}
+
+
+// Files returns the files added to the file set.
+func (s *FileSet) Files() <-chan *File {
+	ch := make(chan *File)
+	go func() {
+		for i := 0; ; i++ {
+			var f *File
+			s.mutex.RLock()
+			if i < len(s.files) {
+				f = s.files[i]
+			}
+			s.mutex.RUnlock()
+			if f == nil {
+				break
+			}
+			ch <- f
+		}
+		close(ch)
+	}()
+	return ch
+}
diff --git a/libgo/go/go/token/position_test.go b/libgo/go/go/token/position_test.go
new file mode 100644
index 000000000..1cffcc3c2
--- /dev/null
+++ b/libgo/go/go/token/position_test.go
@@ -0,0 +1,158 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package token
+
+import (
+	"fmt"
+	"testing"
+)
+
+
+func checkPos(t *testing.T, msg string, p, q Position) {
+	if p.Filename != q.Filename {
+		t.Errorf("%s: expected filename = %q; got %q", msg, q.Filename, p.Filename)
+	}
+	if p.Offset != q.Offset {
+		t.Errorf("%s: expected offset = %d; got %d", msg, q.Offset, p.Offset)
+	}
+	if p.Line != q.Line {
+		t.Errorf("%s: expected line = %d; got %d", msg, q.Line, p.Line)
+	}
+	if p.Column != q.Column {
+		t.Errorf("%s: expected column = %d; got %d", msg, q.Column, p.Column)
+	}
+}
+
+
+func TestNoPos(t *testing.T) {
+	if NoPos.IsValid() {
+		t.Errorf("NoPos should not be valid")
+	}
+	var fset *FileSet
+	checkPos(t, "nil NoPos", fset.Position(NoPos), Position{})
+	fset = NewFileSet()
+	checkPos(t, "fset NoPos", fset.Position(NoPos), Position{})
+}
+
+
+var tests = []struct {
+	filename string
+	size     int
+	lines    []int
+}{
+	{"a", 0, []int{}},
+	{"b", 5, []int{0}},
+	{"c", 10, []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}},
+	{"d", 100, []int{0, 5, 10, 20, 30, 70, 71, 72, 80, 85, 90, 99}},
+	{"e", 777, []int{0, 80, 100, 120, 130, 180, 267, 455, 500, 567, 620}},
+}
+
+
+func linecol(lines []int, offs int) (int, int) {
+	prevLineOffs := 0
+	for line, lineOffs := range lines {
+		if offs < lineOffs {
+			return line, offs - prevLineOffs + 1
+		}
+		prevLineOffs = lineOffs
+	}
+	return len(lines), offs - prevLineOffs + 1
+}
+
+
+func verifyPositions(t *testing.T, fset *FileSet, f *File, lines []int) {
+	for offs := 0; offs < f.Size(); offs++ {
+		p := f.Pos(offs)
+		offs2 := f.Offset(p)
+		if offs2 != offs {
+			t.Errorf("%s, Offset: expected offset %d; got %d", f.Name(), offs, offs2)
+		}
+		line, col := linecol(lines, offs)
+		msg := fmt.Sprintf("%s (offs = %d, p = %d)", f.Name(), offs, p)
+		checkPos(t, msg, f.Position(f.Pos(offs)), Position{f.Name(), offs, line, col})
+		checkPos(t, msg, fset.Position(p), Position{f.Name(), offs, line, col})
+	}
+}
+
+
+func TestPositions(t *testing.T) {
+	const delta = 7 // a non-zero base offset increment
+	fset := NewFileSet()
+	for _, test := range tests {
+		// add file and verify name and size
+		f := fset.AddFile(test.filename, fset.Base()+delta, test.size)
+		if f.Name() != test.filename {
+			t.Errorf("expected filename %q; got %q", test.filename, f.Name())
+		}
+		if f.Size() != test.size {
+			t.Errorf("%s: expected file size %d; got %d", f.Name(), test.size, f.Size())
+		}
+		if fset.File(f.Pos(0)) != f {
+			t.Errorf("%s: f.Pos(0) was not found in f", f.Name())
+		}
+
+		// add lines individually and verify all positions
+		for i, offset := range test.lines {
+			f.AddLine(offset)
+			if f.LineCount() != i+1 {
+				t.Errorf("%s, AddLine: expected line count %d; got %d", f.Name(), i+1, f.LineCount())
+			}
+			// adding the same offset again should be ignored
+			f.AddLine(offset)
+			if f.LineCount() != i+1 {
+				t.Errorf("%s, AddLine: expected unchanged line count %d; got %d", f.Name(), i+1, f.LineCount())
+			}
+			verifyPositions(t, fset, f, test.lines[0:i+1])
+		}
+
+		// add lines at once and verify all positions
+		ok := f.SetLines(test.lines)
+		if !ok {
+			t.Errorf("%s: SetLines failed", f.Name())
+		}
+		if f.LineCount() != len(test.lines) {
+			t.Errorf("%s, SetLines: expected line count %d; got %d", f.Name(), len(test.lines), f.LineCount())
+		}
+		verifyPositions(t, fset, f, test.lines)
+	}
+}
+
+
+func TestLineInfo(t *testing.T) {
+	fset := NewFileSet()
+	f := fset.AddFile("foo", fset.Base(), 500)
+	lines := []int{0, 42, 77, 100, 210, 220, 277, 300, 333, 401}
+	// add lines individually and provide alternative line information
+	for _, offs := range lines {
+		f.AddLine(offs)
+		f.AddLineInfo(offs, "bar", 42)
+	}
+	// verify positions for all offsets
+	for offs := 0; offs <= f.Size(); offs++ {
+		p := f.Pos(offs)
+		_, col := linecol(lines, offs)
+		msg := fmt.Sprintf("%s (offs = %d, p = %d)", f.Name(), offs, p)
+		checkPos(t, msg, f.Position(f.Pos(offs)), Position{"bar", offs, 42, col})
+		checkPos(t, msg, fset.Position(p), Position{"bar", offs, 42, col})
+	}
+}
+
+
+func TestFiles(t *testing.T) {
+	fset := NewFileSet()
+	for i, test := range tests {
+		fset.AddFile(test.filename, fset.Base(), test.size)
+		j := 0
+		for g := range fset.Files() {
+			if g.Name() != tests[j].filename {
+				t.Errorf("expected filename = %s; got %s", tests[j].filename, g.Name())
+			}
+			j++
+		}
+		if j != i+1 {
+			t.Errorf("expected %d files; got %d", i+1, j)
+		}
+	}
+}
diff --git a/libgo/go/go/token/token.go b/libgo/go/go/token/token.go
new file mode 100644
index 000000000..1bd81c1b1
--- /dev/null
+++ b/libgo/go/go/token/token.go
@@ -0,0 +1,320 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This package defines constants representing the lexical
+// tokens of the Go programming language and basic operations
+// on tokens (printing, predicates).
+//
+package token
+
+import "strconv"
+
+
+// Token is the set of lexical tokens of the Go programming language.
+type Token int
+
+// The list of tokens.
+const (
+	// Special tokens
+	ILLEGAL Token = iota
+	EOF
+	COMMENT
+
+	literal_beg
+	// Identifiers and basic type literals
+	// (these tokens stand for classes of literals)
+	IDENT  // main
+	INT    // 12345
+	FLOAT  // 123.45
+	IMAG   // 123.45i
+	CHAR   // 'a'
+	STRING // "abc"
+	literal_end
+
+	operator_beg
+	// Operators and delimiters
+	ADD // +
+	SUB // -
+	MUL // *
+	QUO // /
+	REM // %
+
+	AND     // &
+	OR      // |
+	XOR     // ^
+	SHL     // <<
+	SHR     // >>
+	AND_NOT // &^
+
+	ADD_ASSIGN // +=
+	SUB_ASSIGN // -=
+	MUL_ASSIGN // *=
+	QUO_ASSIGN // /=
+	REM_ASSIGN // %=
+
+	AND_ASSIGN     // &=
+	OR_ASSIGN      // |=
+	XOR_ASSIGN     // ^=
+	SHL_ASSIGN     // <<=
+	SHR_ASSIGN     // >>=
+	AND_NOT_ASSIGN // &^=
+
+	LAND  // &&
+	LOR   // ||
+	ARROW // <-
+	INC   // ++
+	DEC   // --
+
+	EQL    // ==
+	LSS    // <
+	GTR    // >
+	ASSIGN // =
+	NOT    // !
+
+	NEQ      // !=
+	LEQ      // <=
+	GEQ      // >=
+	DEFINE   // :=
+	ELLIPSIS // ...
+
+	LPAREN // (
+	LBRACK // [
+	LBRACE // {
+	COMMA  // ,
+	PERIOD // .
+
+	RPAREN    // )
+	RBRACK    // ]
+	RBRACE    // }
+	SEMICOLON // ;
+	COLON     // :
+	operator_end
+
+	keyword_beg
+	// Keywords
+	BREAK
+	CASE
+	CHAN
+	CONST
+	CONTINUE
+
+	DEFAULT
+	DEFER
+	ELSE
+	FALLTHROUGH
+	FOR
+
+	FUNC
+	GO
+	GOTO
+	IF
+	IMPORT
+
+	INTERFACE
+	MAP
+	PACKAGE
+	RANGE
+	RETURN
+
+	SELECT
+	STRUCT
+	SWITCH
+	TYPE
+	VAR
+	keyword_end
+)
+
+
+// At the moment we have no array literal syntax that lets us describe
+// the index for each element - use a map for now to make sure they are
+// in sync.
+var tokens = map[Token]string{
+	ILLEGAL: "ILLEGAL",
+
+	EOF:     "EOF",
+	COMMENT: "COMMENT",
+
+	IDENT:  "IDENT",
+	INT:    "INT",
+	FLOAT:  "FLOAT",
+	IMAG:   "IMAG",
+	CHAR:   "CHAR",
+	STRING: "STRING",
+
+	ADD: "+",
+	SUB: "-",
+	MUL: "*",
+	QUO: "/",
+	REM: "%",
+
+	AND:     "&",
+	OR:      "|",
+	XOR:     "^",
+	SHL:     "<<",
+	SHR:     ">>",
+	AND_NOT: "&^",
+
+	ADD_ASSIGN: "+=",
+	SUB_ASSIGN: "-=",
+	MUL_ASSIGN: "*=",
+	QUO_ASSIGN: "/=",
+	REM_ASSIGN: "%=",
+
+	AND_ASSIGN:     "&=",
+	OR_ASSIGN:      "|=",
+	XOR_ASSIGN:     "^=",
+	SHL_ASSIGN:     "<<=",
+	SHR_ASSIGN:     ">>=",
+	AND_NOT_ASSIGN: "&^=",
+
+	LAND:  "&&",
+	LOR:   "||",
+	ARROW: "<-",
+	INC:   "++",
+	DEC:   "--",
+
+	EQL:    "==",
+	LSS:    "<",
+	GTR:    ">",
+	ASSIGN: "=",
+	NOT:    "!",
+
+	NEQ:      "!=",
+	LEQ:      "<=",
+	GEQ:      ">=",
+	DEFINE:   ":=",
+	ELLIPSIS: "...",
+
+	LPAREN: "(",
+	LBRACK: "[",
+	LBRACE: "{",
+	COMMA:  ",",
+	PERIOD: ".",
+
+	RPAREN:    ")",
+	RBRACK:    "]",
+	RBRACE:    "}",
+	SEMICOLON: ";",
+	COLON:     ":",
+
+	BREAK:    "break",
+	CASE:     "case",
+	CHAN:     "chan",
+	CONST:    "const",
+	CONTINUE: "continue",
+
+	DEFAULT:     "default",
+	DEFER:       "defer",
+	ELSE:        "else",
+	FALLTHROUGH: "fallthrough",
+	FOR:         "for",
+
+	FUNC:   "func",
+	GO:     "go",
+	GOTO:   "goto",
+	IF:     "if",
+	IMPORT: "import",
+
+	INTERFACE: "interface",
+	MAP:       "map",
+	PACKAGE:   "package",
+	RANGE:     "range",
+	RETURN:    "return",
+
+	SELECT: "select",
+	STRUCT: "struct",
+	SWITCH: "switch",
+	TYPE:   "type",
+	VAR:    "var",
+}
+
+
+// String returns the string corresponding to the token tok.
+// For operators, delimiters, and keywords the string is the actual
+// token character sequence (e.g., for the token ADD, the string is
+// "+"). For all other tokens the string corresponds to the token
+// constant name (e.g. for the token IDENT, the string is "IDENT").
+//
+func (tok Token) String() string {
+	if str, exists := tokens[tok]; exists {
+		return str
+	}
+	return "token(" + strconv.Itoa(int(tok)) + ")"
+}
+
+
+// A set of constants for precedence-based expression parsing.
+// Non-operators have lowest precedence, followed by operators
+// starting with precedence 1 up to unary operators. The highest
+// precedence corresponds serves as "catch-all" precedence for
+// selector, indexing, and other operator and delimiter tokens.
+//
+const (
+	LowestPrec  = 0 // non-operators
+	UnaryPrec   = 7
+	HighestPrec = 8
+)
+
+
+// Precedence returns the operator precedence of the binary
+// operator op. If op is not a binary operator, the result
+// is LowestPrecedence.
+//
+func (op Token) Precedence() int {
+	switch op {
+	case LOR:
+		return 1
+	case LAND:
+		return 2
+	case ARROW:
+		return 3
+	case EQL, NEQ, LSS, LEQ, GTR, GEQ:
+		return 4
+	case ADD, SUB, OR, XOR:
+		return 5
+	case MUL, QUO, REM, SHL, SHR, AND, AND_NOT:
+		return 6
+	}
+	return LowestPrec
+}
+
+
+var keywords map[string]Token
+
+func init() {
+	keywords = make(map[string]Token)
+	for i := keyword_beg + 1; i < keyword_end; i++ {
+		keywords[tokens[i]] = i
+	}
+}
+
+
+// Lookup maps an identifier to its keyword token or IDENT (if not a keyword).
+//
+func Lookup(ident []byte) Token {
+	// TODO Maps with []byte key are illegal because []byte does not
+	//      support == . Should find a more efficient solution eventually.
+	if tok, is_keyword := keywords[string(ident)]; is_keyword {
+		return tok
+	}
+	return IDENT
+}
+
+
+// Predicates
+
+// IsLiteral returns true for tokens corresponding to identifiers
+// and basic type literals; returns false otherwise.
+//
+func (tok Token) IsLiteral() bool { return literal_beg < tok && tok < literal_end }
+
+// IsOperator returns true for tokens corresponding to operators and
+// delimiters; returns false otherwise.
+//
+func (tok Token) IsOperator() bool { return operator_beg < tok && tok < operator_end }
+
+// IsKeyword returns true for tokens corresponding to keywords;
+// returns false otherwise.
+//
+func (tok Token) IsKeyword() bool { return keyword_beg < tok && tok < keyword_end }
diff --git a/libgo/go/go/typechecker/scope.go b/libgo/go/go/typechecker/scope.go
new file mode 100644
index 000000000..114c93ea8
--- /dev/null
+++ b/libgo/go/go/typechecker/scope.go
@@ -0,0 +1,119 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This file implements scope support functions.
+
+package typechecker
+
+import (
+	"fmt"
+	"go/ast"
+	"go/token"
+)
+
+
+func (tc *typechecker) openScope() *ast.Scope {
+	tc.topScope = ast.NewScope(tc.topScope)
+	return tc.topScope
+}
+
+
+func (tc *typechecker) closeScope() {
+	tc.topScope = tc.topScope.Outer
+}
+
+
+// objPos computes the source position of the declaration of an object name.
+// Only required for error reporting, so doesn't have to be fast.
+func objPos(obj *ast.Object) (pos token.Pos) {
+	switch d := obj.Decl.(type) {
+	case *ast.Field:
+		for _, n := range d.Names {
+			if n.Name == obj.Name {
+				return n.Pos()
+			}
+		}
+	case *ast.ValueSpec:
+		for _, n := range d.Names {
+			if n.Name == obj.Name {
+				return n.Pos()
+			}
+		}
+	case *ast.TypeSpec:
+		return d.Name.Pos()
+	case *ast.FuncDecl:
+		return d.Name.Pos()
+	}
+	if debug {
+		fmt.Printf("decl = %T\n", obj.Decl)
+	}
+	panic("unreachable")
+}
+
+
+// declInScope declares an object of a given kind and name in scope and sets the object's Decl and N fields.
+// It returns the newly allocated object. If an object with the same name already exists in scope, an error
+// is reported and the object is not inserted.
+// (Objects with _ name are always inserted into a scope without errors, but they cannot be found.)
+func (tc *typechecker) declInScope(scope *ast.Scope, kind ast.Kind, name *ast.Ident, decl interface{}, n int) *ast.Object {
+	obj := ast.NewObj(kind, name.Name)
+	obj.Decl = decl
+	obj.N = n
+	name.Obj = obj
+	if alt := scope.Insert(obj); alt != obj {
+		tc.Errorf(name.Pos(), "%s already declared at %s", name.Name, objPos(alt))
+	}
+	return obj
+}
+
+
+// decl is the same as declInScope(tc.topScope, ...)
+func (tc *typechecker) decl(kind ast.Kind, name *ast.Ident, decl interface{}, n int) *ast.Object {
+	return tc.declInScope(tc.topScope, kind, name, decl, n)
+}
+
+
+// find returns the object with the given name if visible in the current scope hierarchy.
+// If no such object is found, an error is reported and a bad object is returned instead.
+func (tc *typechecker) find(name *ast.Ident) (obj *ast.Object) {
+	for s := tc.topScope; s != nil && obj == nil; s = s.Outer {
+		obj = s.Lookup(name.Name)
+	}
+	if obj == nil {
+		tc.Errorf(name.Pos(), "%s not declared", name.Name)
+		obj = ast.NewObj(ast.Bad, name.Name)
+	}
+	name.Obj = obj
+	return
+}
+
+
+// findField returns the object with the given name if visible in the type's scope.
+// If no such object is found, an error is reported and a bad object is returned instead.
+func (tc *typechecker) findField(typ *ast.Type, name *ast.Ident) (obj *ast.Object) {
+	// TODO(gri) This is simplistic at the moment and ignores anonymous fields.
+	obj = typ.Scope.Lookup(name.Name)
+	if obj == nil {
+		tc.Errorf(name.Pos(), "%s not declared", name.Name)
+		obj = ast.NewObj(ast.Bad, name.Name)
+	}
+	return
+}
+
+
+// printScope prints the objects in a scope.
+func printScope(scope *ast.Scope) {
+	fmt.Printf("scope %p {", scope)
+	if scope != nil && len(scope.Objects) > 0 {
+		fmt.Println()
+		for _, obj := range scope.Objects {
+			form := "void"
+			if obj.Type != nil {
+				form = obj.Type.Form.String()
+			}
+			fmt.Printf("\t%s\t%s\n", obj.Name, form)
+		}
+	}
+	fmt.Printf("}\n")
+}
diff --git a/libgo/go/go/typechecker/testdata/test0.go b/libgo/go/go/typechecker/testdata/test0.go
new file mode 100644
index 000000000..4e317f214
--- /dev/null
+++ b/libgo/go/go/typechecker/testdata/test0.go
@@ -0,0 +1,94 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// type declarations
+
+package P0
+
+type (
+	B bool
+	I int32
+	A [10]P
+	T struct {
+		x, y P
+	}
+	P *T
+	R *R
+	F func(A) I
+	Y interface {
+		f(A) I
+	}
+	S []P
+	M map[I]F
+	C chan<- I
+)
+
+type (
+	a/* ERROR "illegal cycle" */ a
+	a/* ERROR "already declared" */ int
+
+	b/* ERROR "illegal cycle" */ c
+	c d
+	d e
+	e b /* ERROR "not a type" */
+
+	t *t
+
+	U V
+	V W
+	W *U
+
+	P1 *S2
+	P2 P1
+
+	S1 struct {
+		a, b, c int
+		u, v, a/* ERROR "already declared" */ float
+	}
+	S2/* ERROR "illegal cycle" */ struct {
+		x S2
+	}
+
+	L1 []L1
+	L2 []int
+
+	A1 [10]int
+	A2/* ERROR "illegal cycle" */ [10]A2
+	A3/* ERROR "illegal cycle" */ [10]struct {
+		x A4
+	}
+	A4 [10]A3
+
+	F1 func()
+	F2 func(x, y, z float)
+	F3 func(x, y, x /* ERROR "already declared" */ float)
+	F4 func() (x, y, x /* ERROR "already declared" */ float)
+	F5 func(x int) (x /* ERROR "already declared" */ float)
+
+	I1 interface{}
+	I2 interface {
+		m1()
+	}
+	I3 interface {
+		m1()
+		m1 /* ERROR "already declared" */ ()
+	}
+	I4 interface {
+		m1(x, y, x /* ERROR "already declared" */ float)
+		m2() (x, y, x /* ERROR "already declared" */ float)
+		m3(x int) (x /* ERROR "already declared" */ float)
+	}
+	I5 interface {
+		m1(I5)
+	}
+
+	C1 chan int
+	C2 <-chan int
+	C3 chan<- C3
+
+	M1 map[Last]string
+	M2 map[string]M2
+
+	Last int
+)
diff --git a/libgo/go/go/typechecker/testdata/test1.go b/libgo/go/go/typechecker/testdata/test1.go
new file mode 100644
index 000000000..b0808ee7a
--- /dev/null
+++ b/libgo/go/go/typechecker/testdata/test1.go
@@ -0,0 +1,13 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// const and var declarations
+
+package P1
+
+const (
+	c1         /* ERROR "missing initializer" */
+	c2     int = 0
+	c3, c4 = 0
+)
diff --git a/libgo/go/go/typechecker/testdata/test3.go b/libgo/go/go/typechecker/testdata/test3.go
new file mode 100644
index 000000000..ea35808a0
--- /dev/null
+++ b/libgo/go/go/typechecker/testdata/test3.go
@@ -0,0 +1,38 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package P3
+
+// function and method signatures
+
+func _()                                        {}
+func _()                                        {}
+func _(x, x /* ERROR "already declared" */ int) {}
+
+func f()                                 {}
+func f /* ERROR "already declared" */ () {}
+
+func (*foo /* ERROR "invalid receiver" */ ) m() {}
+func (bar /* ERROR "not a type" */ ) m()        {}
+
+func f1(x, _, _ int) (_, _ float)                              {}
+func f2(x, y, x /* ERROR "already declared" */ int)            {}
+func f3(x, y int) (a, b, x /* ERROR "already declared" */ int) {}
+
+func (x *T) m1()                                 {}
+func (x *T) m1 /* ERROR "already declared" */ () {}
+func (x T) m1 /* ERROR "already declared" */ ()  {}
+func (T) m1 /* ERROR "already declared" */ ()    {}
+
+func (x *T) m2(u, x /* ERROR "already declared" */ int)               {}
+func (x *T) m3(a, b, c int) (u, x /* ERROR "already declared" */ int) {}
+func (T) _(x, x /* ERROR "already declared" */ int)                   {}
+func (T) _() (x, x /* ERROR "already declared" */ int)                {}
+
+//func (PT) _() {}
+
+var bar int
+
+type T struct{}
+type PT (T)
diff --git a/libgo/go/go/typechecker/testdata/test4.go b/libgo/go/go/typechecker/testdata/test4.go
new file mode 100644
index 000000000..bb9aee3ad
--- /dev/null
+++ b/libgo/go/go/typechecker/testdata/test4.go
@@ -0,0 +1,11 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Constant declarations
+
+package P4
+
+const (
+	c0 /* ERROR "missing initializer" */
+)
diff --git a/libgo/go/go/typechecker/typechecker.go b/libgo/go/go/typechecker/typechecker.go
new file mode 100644
index 000000000..e9aefa240
--- /dev/null
+++ b/libgo/go/go/typechecker/typechecker.go
@@ -0,0 +1,484 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// INCOMPLETE PACKAGE.
+// This package implements typechecking of a Go AST.
+// The result of the typecheck is an augmented AST
+// with object and type information for each identifier.
+//
+package typechecker
+
+import (
+	"fmt"
+	"go/ast"
+	"go/token"
+	"go/scanner"
+	"os"
+)
+
+
+// TODO(gri) don't report errors for objects/types that are marked as bad.
+
+
+const debug = true // set for debugging output
+
+
+// An importer takes an import path and returns the data describing the
+// respective package's exported interface. The data format is TBD.
+//
+type Importer func(path string) ([]byte, os.Error)
+
+
+// CheckPackage typechecks a package and augments the AST by setting
+// *ast.Object, *ast.Type, and *ast.Scope fields accordingly. If an
+// importer is provided, it is used to handle imports, otherwise they
+// are ignored (likely leading to typechecking errors).
+//
+// If errors are reported, the AST may be incompletely augmented (fields
+// may be nil) or contain incomplete object, type, or scope information.
+//
+func CheckPackage(fset *token.FileSet, pkg *ast.Package, importer Importer) os.Error {
+	var tc typechecker
+	tc.fset = fset
+	tc.importer = importer
+	tc.checkPackage(pkg)
+	return tc.GetError(scanner.Sorted)
+}
+
+
+// CheckFile typechecks a single file, but otherwise behaves like
+// CheckPackage. If the complete package consists of more than just
+// one file, the file may not typecheck without errors.
+//
+func CheckFile(fset *token.FileSet, file *ast.File, importer Importer) os.Error {
+	// create a single-file dummy package
+	pkg := &ast.Package{file.Name.Name, nil, map[string]*ast.File{fset.Position(file.Name.NamePos).Filename: file}}
+	return CheckPackage(fset, pkg, importer)
+}
+
+
+// ----------------------------------------------------------------------------
+// Typechecker state
+
+type typechecker struct {
+	fset *token.FileSet
+	scanner.ErrorVector
+	importer Importer
+	topScope *ast.Scope           // current top-most scope
+	cyclemap map[*ast.Object]bool // for cycle detection
+	iota     int                  // current value of iota
+}
+
+
+func (tc *typechecker) Errorf(pos token.Pos, format string, args ...interface{}) {
+	tc.Error(tc.fset.Position(pos), fmt.Sprintf(format, args...))
+}
+
+
+func assert(pred bool) {
+	if !pred {
+		panic("internal error")
+	}
+}
+
+
+/*
+Typechecking is done in several phases:
+
+phase 1: declare all global objects; also collect all function and method declarations
+	- all objects have kind, name, decl fields; the decl field permits
+	  quick lookup of an object's declaration
+	- constant objects have an iota value
+	- type objects have unresolved types with empty scopes, all others have nil types
+	- report global double declarations
+
+phase 2: bind methods to their receiver base types
+	- received base types must be declared in the package, thus for
+	  each method a corresponding (unresolved) type must exist
+	- report method double declarations and errors with base types
+
+phase 3: resolve all global objects
+	- sequentially iterate through all objects in the global scope
+	- resolve types for all unresolved types and assign types to
+	  all attached methods
+	- assign types to all other objects, possibly by evaluating
+	  constant and initializer expressions
+	- resolution may recurse; a cyclemap is used to detect cycles
+	- report global typing errors
+
+phase 4: sequentially typecheck function and method bodies
+	- all global objects are declared and have types and values;
+	  all methods have types
+	- sequentially process statements in each body; any object
+	  referred to must be fully defined at this point
+	- report local typing errors
+*/
+
+func (tc *typechecker) checkPackage(pkg *ast.Package) {
+	// setup package scope
+	tc.topScope = Universe
+	tc.openScope()
+	defer tc.closeScope()
+
+	// TODO(gri) there's no file scope at the moment since we ignore imports
+
+	// phase 1: declare all global objects; also collect all function and method declarations
+	var funcs []*ast.FuncDecl
+	for _, file := range pkg.Files {
+		for _, decl := range file.Decls {
+			tc.declGlobal(decl)
+			if f, isFunc := decl.(*ast.FuncDecl); isFunc {
+				funcs = append(funcs, f)
+			}
+		}
+	}
+
+	// phase 2: bind methods to their receiver base types
+	for _, m := range funcs {
+		if m.Recv != nil {
+			tc.bindMethod(m)
+		}
+	}
+
+	// phase 3: resolve all global objects
+	// (note that objects with _ name are also in the scope)
+	tc.cyclemap = make(map[*ast.Object]bool)
+	for _, obj := range tc.topScope.Objects {
+		tc.resolve(obj)
+	}
+	assert(len(tc.cyclemap) == 0)
+
+	// 4: sequentially typecheck function and method bodies
+	for _, f := range funcs {
+		tc.checkBlock(f.Body.List, f.Name.Obj.Type)
+	}
+
+	pkg.Scope = tc.topScope
+}
+
+
+func (tc *typechecker) declGlobal(global ast.Decl) {
+	switch d := global.(type) {
+	case *ast.BadDecl:
+		// ignore
+
+	case *ast.GenDecl:
+		iota := 0
+		var prev *ast.ValueSpec
+		for _, spec := range d.Specs {
+			switch s := spec.(type) {
+			case *ast.ImportSpec:
+				// TODO(gri) imports go into file scope
+			case *ast.ValueSpec:
+				switch d.Tok {
+				case token.CONST:
+					if s.Values == nil {
+						// create a new spec with type and values from the previous one
+						if prev != nil {
+							s = &ast.ValueSpec{s.Doc, s.Names, prev.Type, prev.Values, s.Comment}
+						} else {
+							// TODO(gri) this should probably go into the const decl code
+							tc.Errorf(s.Pos(), "missing initializer for const %s", s.Names[0].Name)
+						}
+					}
+					for _, name := range s.Names {
+						tc.decl(ast.Con, name, s, iota)
+					}
+				case token.VAR:
+					for _, name := range s.Names {
+						tc.decl(ast.Var, name, s, 0)
+					}
+				default:
+					panic("unreachable")
+				}
+				prev = s
+				iota++
+			case *ast.TypeSpec:
+				obj := tc.decl(ast.Typ, s.Name, s, 0)
+				// give all type objects an unresolved type so
+				// that we can collect methods in the type scope
+				typ := ast.NewType(ast.Unresolved)
+				obj.Type = typ
+				typ.Obj = obj
+			default:
+				panic("unreachable")
+			}
+		}
+
+	case *ast.FuncDecl:
+		if d.Recv == nil {
+			tc.decl(ast.Fun, d.Name, d, 0)
+		}
+
+	default:
+		panic("unreachable")
+	}
+}
+
+
+// If x is of the form *T, deref returns T, otherwise it returns x.
+func deref(x ast.Expr) ast.Expr {
+	if p, isPtr := x.(*ast.StarExpr); isPtr {
+		x = p.X
+	}
+	return x
+}
+
+
+func (tc *typechecker) bindMethod(method *ast.FuncDecl) {
+	// a method is declared in the receiver base type's scope
+	var scope *ast.Scope
+	base := deref(method.Recv.List[0].Type)
+	if name, isIdent := base.(*ast.Ident); isIdent {
+		// if base is not an *ast.Ident, we had a syntax
+		// error and the parser reported an error already
+		obj := tc.topScope.Lookup(name.Name)
+		if obj == nil {
+			tc.Errorf(name.Pos(), "invalid receiver: %s is not declared in this package", name.Name)
+		} else if obj.Kind != ast.Typ {
+			tc.Errorf(name.Pos(), "invalid receiver: %s is not a type", name.Name)
+		} else {
+			typ := obj.Type
+			assert(typ.Form == ast.Unresolved)
+			scope = typ.Scope
+		}
+	}
+	if scope == nil {
+		// no receiver type found; use a dummy scope
+		// (we still want to type-check the method
+		// body, so make sure there is a name object
+		// and type)
+		// TODO(gri) should we record the scope so
+		// that we don't lose the receiver for type-
+		// checking of the method body?
+		scope = ast.NewScope(nil)
+	}
+	tc.declInScope(scope, ast.Fun, method.Name, method, 0)
+}
+
+
+func (tc *typechecker) resolve(obj *ast.Object) {
+	// check for declaration cycles
+	if tc.cyclemap[obj] {
+		tc.Errorf(objPos(obj), "illegal cycle in declaration of %s", obj.Name)
+		obj.Kind = ast.Bad
+		return
+	}
+	tc.cyclemap[obj] = true
+	defer func() {
+		tc.cyclemap[obj] = false, false
+	}()
+
+	// resolve non-type objects
+	typ := obj.Type
+	if typ == nil {
+		switch obj.Kind {
+		case ast.Bad:
+			// ignore
+
+		case ast.Con:
+			tc.declConst(obj)
+
+		case ast.Var:
+			tc.declVar(obj)
+			//obj.Type = tc.typeFor(nil, obj.Decl.(*ast.ValueSpec).Type, false)
+
+		case ast.Fun:
+			obj.Type = ast.NewType(ast.Function)
+			t := obj.Decl.(*ast.FuncDecl).Type
+			tc.declSignature(obj.Type, nil, t.Params, t.Results)
+
+		default:
+			// type objects have non-nil types when resolve is called
+			if debug {
+				fmt.Printf("kind = %s\n", obj.Kind)
+			}
+			panic("unreachable")
+		}
+		return
+	}
+
+	// resolve type objects
+	if typ.Form == ast.Unresolved {
+		tc.typeFor(typ, typ.Obj.Decl.(*ast.TypeSpec).Type, false)
+
+		// provide types for all methods
+		for _, obj := range typ.Scope.Objects {
+			if obj.Kind == ast.Fun {
+				assert(obj.Type == nil)
+				obj.Type = ast.NewType(ast.Method)
+				f := obj.Decl.(*ast.FuncDecl)
+				t := f.Type
+				tc.declSignature(obj.Type, f.Recv, t.Params, t.Results)
+			}
+		}
+	}
+}
+
+
+func (tc *typechecker) checkBlock(body []ast.Stmt, ftype *ast.Type) {
+	tc.openScope()
+	defer tc.closeScope()
+
+	// inject function/method parameters into block scope, if any
+	if ftype != nil {
+		for _, par := range ftype.Params.Objects {
+			obj := tc.topScope.Insert(par)
+			assert(obj == par) // ftype has no double declarations
+		}
+	}
+
+	for _, stmt := range body {
+		tc.checkStmt(stmt)
+	}
+}
+
+
+// ----------------------------------------------------------------------------
+// Types
+
+// unparen removes parentheses around x, if any.
+func unparen(x ast.Expr) ast.Expr {
+	if ux, hasParens := x.(*ast.ParenExpr); hasParens {
+		return unparen(ux.X)
+	}
+	return x
+}
+
+
+func (tc *typechecker) declFields(scope *ast.Scope, fields *ast.FieldList, ref bool) (n uint) {
+	if fields != nil {
+		for _, f := range fields.List {
+			typ := tc.typeFor(nil, f.Type, ref)
+			for _, name := range f.Names {
+				fld := tc.declInScope(scope, ast.Var, name, f, 0)
+				fld.Type = typ
+				n++
+			}
+		}
+	}
+	return n
+}
+
+
+func (tc *typechecker) declSignature(typ *ast.Type, recv, params, results *ast.FieldList) {
+	assert((typ.Form == ast.Method) == (recv != nil))
+	typ.Params = ast.NewScope(nil)
+	tc.declFields(typ.Params, recv, true)
+	tc.declFields(typ.Params, params, true)
+	typ.N = tc.declFields(typ.Params, results, true)
+}
+
+
+func (tc *typechecker) typeFor(def *ast.Type, x ast.Expr, ref bool) (typ *ast.Type) {
+	x = unparen(x)
+
+	// type name
+	if t, isIdent := x.(*ast.Ident); isIdent {
+		obj := tc.find(t)
+
+		if obj.Kind != ast.Typ {
+			tc.Errorf(t.Pos(), "%s is not a type", t.Name)
+			if def == nil {
+				typ = ast.NewType(ast.BadType)
+			} else {
+				typ = def
+				typ.Form = ast.BadType
+			}
+			typ.Expr = x
+			return
+		}
+
+		if !ref {
+			tc.resolve(obj) // check for cycles even if type resolved
+		}
+		typ = obj.Type
+
+		if def != nil {
+			// new type declaration: copy type structure
+			def.Form = typ.Form
+			def.N = typ.N
+			def.Key, def.Elt = typ.Key, typ.Elt
+			def.Params = typ.Params
+			def.Expr = x
+			typ = def
+		}
+		return
+	}
+
+	// type literal
+	typ = def
+	if typ == nil {
+		typ = ast.NewType(ast.BadType)
+	}
+	typ.Expr = x
+
+	switch t := x.(type) {
+	case *ast.SelectorExpr:
+		if debug {
+			fmt.Println("qualified identifier unimplemented")
+		}
+		typ.Form = ast.BadType
+
+	case *ast.StarExpr:
+		typ.Form = ast.Pointer
+		typ.Elt = tc.typeFor(nil, t.X, true)
+
+	case *ast.ArrayType:
+		if t.Len != nil {
+			typ.Form = ast.Array
+			// TODO(gri) compute the real length
+			// (this may call resolve recursively)
+			(*typ).N = 42
+		} else {
+			typ.Form = ast.Slice
+		}
+		typ.Elt = tc.typeFor(nil, t.Elt, t.Len == nil)
+
+	case *ast.StructType:
+		typ.Form = ast.Struct
+		tc.declFields(typ.Scope, t.Fields, false)
+
+	case *ast.FuncType:
+		typ.Form = ast.Function
+		tc.declSignature(typ, nil, t.Params, t.Results)
+
+	case *ast.InterfaceType:
+		typ.Form = ast.Interface
+		tc.declFields(typ.Scope, t.Methods, true)
+
+	case *ast.MapType:
+		typ.Form = ast.Map
+		typ.Key = tc.typeFor(nil, t.Key, true)
+		typ.Elt = tc.typeFor(nil, t.Value, true)
+
+	case *ast.ChanType:
+		typ.Form = ast.Channel
+		typ.N = uint(t.Dir)
+		typ.Elt = tc.typeFor(nil, t.Value, true)
+
+	default:
+		if debug {
+			fmt.Printf("x is %T\n", x)
+		}
+		panic("unreachable")
+	}
+
+	return
+}
+
+
+// ----------------------------------------------------------------------------
+// TODO(gri) implement these place holders
+
+func (tc *typechecker) declConst(*ast.Object) {
+}
+
+
+func (tc *typechecker) declVar(*ast.Object) {
+}
+
+
+func (tc *typechecker) checkStmt(ast.Stmt) {
+}
diff --git a/libgo/go/go/typechecker/typechecker_test.go b/libgo/go/go/typechecker/typechecker_test.go
new file mode 100644
index 000000000..33f4a6223
--- /dev/null
+++ b/libgo/go/go/typechecker/typechecker_test.go
@@ -0,0 +1,168 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This file implements a simple typechecker test harness. Packages found
+// in the testDir directory are typechecked. Error messages reported by
+// the typechecker are compared against the error messages expected for
+// the test files.
+//
+// Expected errors are indicated in the test files by putting a comment
+// of the form /* ERROR "rx" */ immediately following an offending token.
+// The harness will verify that an error matching the regular expression
+// rx is reported at that source position. Consecutive comments may be
+// used to indicate multiple errors for the same token position.
+//
+// For instance, the following test file indicates that a "not declared"
+// error should be reported for the undeclared variable x:
+//
+//	package P0
+//	func f() {
+//		_ = x /* ERROR "not declared" */ + 1
+//	}
+// 
+// If the -pkg flag is set, only packages with package names matching
+// the regular expression provided via the flag value are tested.
+
+package typechecker
+
+import (
+	"flag"
+	"fmt"
+	"go/ast"
+	"go/parser"
+	"go/scanner"
+	"go/token"
+	"io/ioutil"
+	"os"
+	"regexp"
+	"sort"
+	"strings"
+	"testing"
+)
+
+
+const testDir = "./testdata" // location of test packages
+
+var fset = token.NewFileSet()
+
+var (
+	pkgPat = flag.String("pkg", ".*", "regular expression to select test packages by package name")
+	trace  = flag.Bool("trace", false, "print package names")
+)
+
+
+// ERROR comments must be of the form /* ERROR "rx" */ and rx is
+// a regular expression that matches the expected error message.
+var errRx = regexp.MustCompile(`^/\* *ERROR *"([^"]*)" *\*/$`)
+
+// expectedErrors collects the regular expressions of ERROR comments
+// found in the package files of pkg and returns them in sorted order
+// (by filename and position).
+func expectedErrors(t *testing.T, pkg *ast.Package) (list scanner.ErrorList) {
+	// scan all package files
+	for filename := range pkg.Files {
+		src, err := ioutil.ReadFile(filename)
+		if err != nil {
+			t.Fatalf("expectedErrors(%s): %v", pkg.Name, err)
+		}
+
+		var s scanner.Scanner
+		file := fset.AddFile(filename, fset.Base(), len(src))
+		s.Init(file, src, nil, scanner.ScanComments)
+		var prev token.Pos // position of last non-comment token
+	loop:
+		for {
+			pos, tok, lit := s.Scan()
+			switch tok {
+			case token.EOF:
+				break loop
+			case token.COMMENT:
+				s := errRx.FindSubmatch(lit)
+				if len(s) == 2 {
+					list = append(list, &scanner.Error{fset.Position(prev), string(s[1])})
+				}
+			default:
+				prev = pos
+			}
+		}
+	}
+	sort.Sort(list) // multiple files may not be sorted
+	return
+}
+
+
+func testFilter(f *os.FileInfo) bool {
+	return strings.HasSuffix(f.Name, ".go") && f.Name[0] != '.'
+}
+
+
+func checkError(t *testing.T, expected, found *scanner.Error) {
+	rx, err := regexp.Compile(expected.Msg)
+	if err != nil {
+		t.Errorf("%s: %v", expected.Pos, err)
+		return
+	}
+
+	match := rx.MatchString(found.Msg)
+
+	if expected.Pos.Offset != found.Pos.Offset {
+		if match {
+			t.Errorf("%s: expected error should have been at %s", expected.Pos, found.Pos)
+		} else {
+			t.Errorf("%s: error matching %q expected", expected.Pos, expected.Msg)
+			return
+		}
+	}
+
+	if !match {
+		t.Errorf("%s: %q does not match %q", expected.Pos, expected.Msg, found.Msg)
+	}
+}
+
+
+func TestTypeCheck(t *testing.T) {
+	flag.Parse()
+	pkgRx, err := regexp.Compile(*pkgPat)
+	if err != nil {
+		t.Fatalf("illegal flag value %q: %s", *pkgPat, err)
+	}
+
+	pkgs, err := parser.ParseDir(fset, testDir, testFilter, 0)
+	if err != nil {
+		scanner.PrintError(os.Stderr, err)
+		t.Fatalf("packages in %s contain syntax errors", testDir)
+	}
+
+	for _, pkg := range pkgs {
+		if !pkgRx.MatchString(pkg.Name) {
+			continue // only test selected packages
+		}
+
+		if *trace {
+			fmt.Println(pkg.Name)
+		}
+
+		xlist := expectedErrors(t, pkg)
+		err := CheckPackage(fset, pkg, nil)
+		if err != nil {
+			if elist, ok := err.(scanner.ErrorList); ok {
+				// verify that errors match
+				for i := 0; i < len(xlist) && i < len(elist); i++ {
+					checkError(t, xlist[i], elist[i])
+				}
+				// the correct number or errors must have been found
+				if len(xlist) != len(elist) {
+					fmt.Fprintf(os.Stderr, "%s\n", pkg.Name)
+					scanner.PrintError(os.Stderr, elist)
+					fmt.Fprintln(os.Stderr)
+					t.Errorf("TypeCheck(%s): %d errors expected but %d reported", pkg.Name, len(xlist), len(elist))
+				}
+			} else {
+				t.Errorf("TypeCheck(%s): %v", pkg.Name, err)
+			}
+		} else if len(xlist) > 0 {
+			t.Errorf("TypeCheck(%s): %d errors expected but 0 reported", pkg.Name, len(xlist))
+		}
+	}
+}
diff --git a/libgo/go/go/typechecker/universe.go b/libgo/go/go/typechecker/universe.go
new file mode 100644
index 000000000..db950737f
--- /dev/null
+++ b/libgo/go/go/typechecker/universe.go
@@ -0,0 +1,38 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package typechecker
+
+import "go/ast"
+
+// TODO(gri) should this be in package ast?
+
+// The Universe scope contains all predeclared identifiers.
+var Universe *ast.Scope
+
+
+func def(obj *ast.Object) {
+	alt := Universe.Insert(obj)
+	if alt != obj {
+		panic("object declared twice")
+	}
+}
+
+
+func init() {
+	Universe = ast.NewScope(nil)
+
+	// basic types
+	for n, name := range ast.BasicTypes {
+		typ := ast.NewType(ast.Basic)
+		typ.N = n
+		obj := ast.NewObj(ast.Typ, name)
+		obj.Type = typ
+		typ.Obj = obj
+		def(obj)
+	}
+
+	// built-in functions
+	// TODO(gri) implement this
+}
diff --git a/libgo/go/gob/codec_test.go b/libgo/go/gob/codec_test.go
new file mode 100644
index 000000000..af941c629
--- /dev/null
+++ b/libgo/go/gob/codec_test.go
@@ -0,0 +1,1355 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package gob
+
+import (
+	"bytes"
+	"math"
+	"os"
+	"reflect"
+	"strings"
+	"testing"
+	"unsafe"
+)
+
+// Guarantee encoding format by comparing some encodings to hand-written values
+type EncodeT struct {
+	x uint64
+	b []byte
+}
+
+var encodeT = []EncodeT{
+	{0x00, []byte{0x00}},
+	{0x0F, []byte{0x0F}},
+	{0xFF, []byte{0xFF, 0xFF}},
+	{0xFFFF, []byte{0xFE, 0xFF, 0xFF}},
+	{0xFFFFFF, []byte{0xFD, 0xFF, 0xFF, 0xFF}},
+	{0xFFFFFFFF, []byte{0xFC, 0xFF, 0xFF, 0xFF, 0xFF}},
+	{0xFFFFFFFFFF, []byte{0xFB, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}},
+	{0xFFFFFFFFFFFF, []byte{0xFA, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}},
+	{0xFFFFFFFFFFFFFF, []byte{0xF9, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}},
+	{0xFFFFFFFFFFFFFFFF, []byte{0xF8, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}},
+	{0x1111, []byte{0xFE, 0x11, 0x11}},
+	{0x1111111111111111, []byte{0xF8, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}},
+	{0x8888888888888888, []byte{0xF8, 0x88, 0x88, 0x88, 0x88, 0x88, 0x88, 0x88, 0x88}},
+	{1 << 63, []byte{0xF8, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}},
+}
+
+// testError is meant to be used as a deferred function to turn a panic(gobError) into a
+// plain test.Error call.
+func testError(t *testing.T) {
+	if e := recover(); e != nil {
+		t.Error(e.(gobError).Error) // Will re-panic if not one of our errors, such as a runtime error.
+	}
+	return
+}
+
+// Test basic encode/decode routines for unsigned integers
+func TestUintCodec(t *testing.T) {
+	defer testError(t)
+	b := new(bytes.Buffer)
+	encState := newEncoderState(nil, b)
+	for _, tt := range encodeT {
+		b.Reset()
+		encState.encodeUint(tt.x)
+		if !bytes.Equal(tt.b, b.Bytes()) {
+			t.Errorf("encodeUint: %#x encode: expected % x got % x", tt.x, tt.b, b.Bytes())
+		}
+	}
+	decState := newDecodeState(nil, &b)
+	for u := uint64(0); ; u = (u + 1) * 7 {
+		b.Reset()
+		encState.encodeUint(u)
+		v := decState.decodeUint()
+		if u != v {
+			t.Errorf("Encode/Decode: sent %#x received %#x", u, v)
+		}
+		if u&(1<<63) != 0 {
+			break
+		}
+	}
+}
+
+func verifyInt(i int64, t *testing.T) {
+	defer testError(t)
+	var b = new(bytes.Buffer)
+	encState := newEncoderState(nil, b)
+	encState.encodeInt(i)
+	decState := newDecodeState(nil, &b)
+	decState.buf = make([]byte, 8)
+	j := decState.decodeInt()
+	if i != j {
+		t.Errorf("Encode/Decode: sent %#x received %#x", uint64(i), uint64(j))
+	}
+}
+
+// Test basic encode/decode routines for signed integers
+func TestIntCodec(t *testing.T) {
+	for u := uint64(0); ; u = (u + 1) * 7 {
+		// Do positive and negative values
+		i := int64(u)
+		verifyInt(i, t)
+		verifyInt(-i, t)
+		verifyInt(^i, t)
+		if u&(1<<63) != 0 {
+			break
+		}
+	}
+	verifyInt(-1<<63, t) // a tricky case
+}
+
+// The result of encoding a true boolean with field number 7
+var boolResult = []byte{0x07, 0x01}
+// The result of encoding a number 17 with field number 7
+var signedResult = []byte{0x07, 2 * 17}
+var unsignedResult = []byte{0x07, 17}
+var floatResult = []byte{0x07, 0xFE, 0x31, 0x40}
+// The result of encoding a number 17+19i with field number 7
+var complexResult = []byte{0x07, 0xFE, 0x31, 0x40, 0xFE, 0x33, 0x40}
+// The result of encoding "hello" with field number 7
+var bytesResult = []byte{0x07, 0x05, 'h', 'e', 'l', 'l', 'o'}
+
+func newencoderState(b *bytes.Buffer) *encoderState {
+	b.Reset()
+	state := newEncoderState(nil, b)
+	state.fieldnum = -1
+	return state
+}
+
+// Test instruction execution for encoding.
+// Do not run the machine yet; instead do individual instructions crafted by hand.
+func TestScalarEncInstructions(t *testing.T) {
+	var b = new(bytes.Buffer)
+
+	// bool
+	{
+		data := struct{ a bool }{true}
+		instr := &encInstr{encBool, 6, 0, 0}
+		state := newencoderState(b)
+		instr.op(instr, state, unsafe.Pointer(&data))
+		if !bytes.Equal(boolResult, b.Bytes()) {
+			t.Errorf("bool enc instructions: expected % x got % x", boolResult, b.Bytes())
+		}
+	}
+
+	// int
+	{
+		b.Reset()
+		data := struct{ a int }{17}
+		instr := &encInstr{encInt, 6, 0, 0}
+		state := newencoderState(b)
+		instr.op(instr, state, unsafe.Pointer(&data))
+		if !bytes.Equal(signedResult, b.Bytes()) {
+			t.Errorf("int enc instructions: expected % x got % x", signedResult, b.Bytes())
+		}
+	}
+
+	// uint
+	{
+		b.Reset()
+		data := struct{ a uint }{17}
+		instr := &encInstr{encUint, 6, 0, 0}
+		state := newencoderState(b)
+		instr.op(instr, state, unsafe.Pointer(&data))
+		if !bytes.Equal(unsignedResult, b.Bytes()) {
+			t.Errorf("uint enc instructions: expected % x got % x", unsignedResult, b.Bytes())
+		}
+	}
+
+	// int8
+	{
+		b.Reset()
+		data := struct{ a int8 }{17}
+		instr := &encInstr{encInt8, 6, 0, 0}
+		state := newencoderState(b)
+		instr.op(instr, state, unsafe.Pointer(&data))
+		if !bytes.Equal(signedResult, b.Bytes()) {
+			t.Errorf("int8 enc instructions: expected % x got % x", signedResult, b.Bytes())
+		}
+	}
+
+	// uint8
+	{
+		b.Reset()
+		data := struct{ a uint8 }{17}
+		instr := &encInstr{encUint8, 6, 0, 0}
+		state := newencoderState(b)
+		instr.op(instr, state, unsafe.Pointer(&data))
+		if !bytes.Equal(unsignedResult, b.Bytes()) {
+			t.Errorf("uint8 enc instructions: expected % x got % x", unsignedResult, b.Bytes())
+		}
+	}
+
+	// int16
+	{
+		b.Reset()
+		data := struct{ a int16 }{17}
+		instr := &encInstr{encInt16, 6, 0, 0}
+		state := newencoderState(b)
+		instr.op(instr, state, unsafe.Pointer(&data))
+		if !bytes.Equal(signedResult, b.Bytes()) {
+			t.Errorf("int16 enc instructions: expected % x got % x", signedResult, b.Bytes())
+		}
+	}
+
+	// uint16
+	{
+		b.Reset()
+		data := struct{ a uint16 }{17}
+		instr := &encInstr{encUint16, 6, 0, 0}
+		state := newencoderState(b)
+		instr.op(instr, state, unsafe.Pointer(&data))
+		if !bytes.Equal(unsignedResult, b.Bytes()) {
+			t.Errorf("uint16 enc instructions: expected % x got % x", unsignedResult, b.Bytes())
+		}
+	}
+
+	// int32
+	{
+		b.Reset()
+		data := struct{ a int32 }{17}
+		instr := &encInstr{encInt32, 6, 0, 0}
+		state := newencoderState(b)
+		instr.op(instr, state, unsafe.Pointer(&data))
+		if !bytes.Equal(signedResult, b.Bytes()) {
+			t.Errorf("int32 enc instructions: expected % x got % x", signedResult, b.Bytes())
+		}
+	}
+
+	// uint32
+	{
+		b.Reset()
+		data := struct{ a uint32 }{17}
+		instr := &encInstr{encUint32, 6, 0, 0}
+		state := newencoderState(b)
+		instr.op(instr, state, unsafe.Pointer(&data))
+		if !bytes.Equal(unsignedResult, b.Bytes()) {
+			t.Errorf("uint32 enc instructions: expected % x got % x", unsignedResult, b.Bytes())
+		}
+	}
+
+	// int64
+	{
+		b.Reset()
+		data := struct{ a int64 }{17}
+		instr := &encInstr{encInt64, 6, 0, 0}
+		state := newencoderState(b)
+		instr.op(instr, state, unsafe.Pointer(&data))
+		if !bytes.Equal(signedResult, b.Bytes()) {
+			t.Errorf("int64 enc instructions: expected % x got % x", signedResult, b.Bytes())
+		}
+	}
+
+	// uint64
+	{
+		b.Reset()
+		data := struct{ a uint64 }{17}
+		instr := &encInstr{encUint64, 6, 0, 0}
+		state := newencoderState(b)
+		instr.op(instr, state, unsafe.Pointer(&data))
+		if !bytes.Equal(unsignedResult, b.Bytes()) {
+			t.Errorf("uint64 enc instructions: expected % x got % x", unsignedResult, b.Bytes())
+		}
+	}
+
+	// float32
+	{
+		b.Reset()
+		data := struct{ a float32 }{17}
+		instr := &encInstr{encFloat32, 6, 0, 0}
+		state := newencoderState(b)
+		instr.op(instr, state, unsafe.Pointer(&data))
+		if !bytes.Equal(floatResult, b.Bytes()) {
+			t.Errorf("float32 enc instructions: expected % x got % x", floatResult, b.Bytes())
+		}
+	}
+
+	// float64
+	{
+		b.Reset()
+		data := struct{ a float64 }{17}
+		instr := &encInstr{encFloat64, 6, 0, 0}
+		state := newencoderState(b)
+		instr.op(instr, state, unsafe.Pointer(&data))
+		if !bytes.Equal(floatResult, b.Bytes()) {
+			t.Errorf("float64 enc instructions: expected % x got % x", floatResult, b.Bytes())
+		}
+	}
+
+	// bytes == []uint8
+	{
+		b.Reset()
+		data := struct{ a []byte }{[]byte("hello")}
+		instr := &encInstr{encUint8Array, 6, 0, 0}
+		state := newencoderState(b)
+		instr.op(instr, state, unsafe.Pointer(&data))
+		if !bytes.Equal(bytesResult, b.Bytes()) {
+			t.Errorf("bytes enc instructions: expected % x got % x", bytesResult, b.Bytes())
+		}
+	}
+
+	// string
+	{
+		b.Reset()
+		data := struct{ a string }{"hello"}
+		instr := &encInstr{encString, 6, 0, 0}
+		state := newencoderState(b)
+		instr.op(instr, state, unsafe.Pointer(&data))
+		if !bytes.Equal(bytesResult, b.Bytes()) {
+			t.Errorf("string enc instructions: expected % x got % x", bytesResult, b.Bytes())
+		}
+	}
+}
+
+func execDec(typ string, instr *decInstr, state *decodeState, t *testing.T, p unsafe.Pointer) {
+	defer testError(t)
+	v := int(state.decodeUint())
+	if v+state.fieldnum != 6 {
+		t.Fatalf("decoding field number %d, got %d", 6, v+state.fieldnum)
+	}
+	instr.op(instr, state, decIndirect(p, instr.indir))
+	state.fieldnum = 6
+}
+
+func newDecodeStateFromData(data []byte) *decodeState {
+	b := bytes.NewBuffer(data)
+	state := newDecodeState(nil, &b)
+	state.fieldnum = -1
+	return state
+}
+
+// Test instruction execution for decoding.
+// Do not run the machine yet; instead do individual instructions crafted by hand.
+func TestScalarDecInstructions(t *testing.T) {
+	ovfl := os.ErrorString("overflow")
+
+	// bool
+	{
+		var data struct {
+			a bool
+		}
+		instr := &decInstr{decBool, 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(boolResult)
+		execDec("bool", instr, state, t, unsafe.Pointer(&data))
+		if data.a != true {
+			t.Errorf("bool a = %v not true", data.a)
+		}
+	}
+	// int
+	{
+		var data struct {
+			a int
+		}
+		instr := &decInstr{decOpMap[reflect.Int], 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(signedResult)
+		execDec("int", instr, state, t, unsafe.Pointer(&data))
+		if data.a != 17 {
+			t.Errorf("int a = %v not 17", data.a)
+		}
+	}
+
+	// uint
+	{
+		var data struct {
+			a uint
+		}
+		instr := &decInstr{decOpMap[reflect.Uint], 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(unsignedResult)
+		execDec("uint", instr, state, t, unsafe.Pointer(&data))
+		if data.a != 17 {
+			t.Errorf("uint a = %v not 17", data.a)
+		}
+	}
+
+	// int8
+	{
+		var data struct {
+			a int8
+		}
+		instr := &decInstr{decInt8, 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(signedResult)
+		execDec("int8", instr, state, t, unsafe.Pointer(&data))
+		if data.a != 17 {
+			t.Errorf("int8 a = %v not 17", data.a)
+		}
+	}
+
+	// uint8
+	{
+		var data struct {
+			a uint8
+		}
+		instr := &decInstr{decUint8, 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(unsignedResult)
+		execDec("uint8", instr, state, t, unsafe.Pointer(&data))
+		if data.a != 17 {
+			t.Errorf("uint8 a = %v not 17", data.a)
+		}
+	}
+
+	// int16
+	{
+		var data struct {
+			a int16
+		}
+		instr := &decInstr{decInt16, 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(signedResult)
+		execDec("int16", instr, state, t, unsafe.Pointer(&data))
+		if data.a != 17 {
+			t.Errorf("int16 a = %v not 17", data.a)
+		}
+	}
+
+	// uint16
+	{
+		var data struct {
+			a uint16
+		}
+		instr := &decInstr{decUint16, 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(unsignedResult)
+		execDec("uint16", instr, state, t, unsafe.Pointer(&data))
+		if data.a != 17 {
+			t.Errorf("uint16 a = %v not 17", data.a)
+		}
+	}
+
+	// int32
+	{
+		var data struct {
+			a int32
+		}
+		instr := &decInstr{decInt32, 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(signedResult)
+		execDec("int32", instr, state, t, unsafe.Pointer(&data))
+		if data.a != 17 {
+			t.Errorf("int32 a = %v not 17", data.a)
+		}
+	}
+
+	// uint32
+	{
+		var data struct {
+			a uint32
+		}
+		instr := &decInstr{decUint32, 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(unsignedResult)
+		execDec("uint32", instr, state, t, unsafe.Pointer(&data))
+		if data.a != 17 {
+			t.Errorf("uint32 a = %v not 17", data.a)
+		}
+	}
+
+	// uintptr
+	{
+		var data struct {
+			a uintptr
+		}
+		instr := &decInstr{decOpMap[reflect.Uintptr], 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(unsignedResult)
+		execDec("uintptr", instr, state, t, unsafe.Pointer(&data))
+		if data.a != 17 {
+			t.Errorf("uintptr a = %v not 17", data.a)
+		}
+	}
+
+	// int64
+	{
+		var data struct {
+			a int64
+		}
+		instr := &decInstr{decInt64, 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(signedResult)
+		execDec("int64", instr, state, t, unsafe.Pointer(&data))
+		if data.a != 17 {
+			t.Errorf("int64 a = %v not 17", data.a)
+		}
+	}
+
+	// uint64
+	{
+		var data struct {
+			a uint64
+		}
+		instr := &decInstr{decUint64, 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(unsignedResult)
+		execDec("uint64", instr, state, t, unsafe.Pointer(&data))
+		if data.a != 17 {
+			t.Errorf("uint64 a = %v not 17", data.a)
+		}
+	}
+
+	// float32
+	{
+		var data struct {
+			a float32
+		}
+		instr := &decInstr{decFloat32, 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(floatResult)
+		execDec("float32", instr, state, t, unsafe.Pointer(&data))
+		if data.a != 17 {
+			t.Errorf("float32 a = %v not 17", data.a)
+		}
+	}
+
+	// float64
+	{
+		var data struct {
+			a float64
+		}
+		instr := &decInstr{decFloat64, 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(floatResult)
+		execDec("float64", instr, state, t, unsafe.Pointer(&data))
+		if data.a != 17 {
+			t.Errorf("float64 a = %v not 17", data.a)
+		}
+	}
+
+	// complex64
+	{
+		var data struct {
+			a complex64
+		}
+		instr := &decInstr{decOpMap[reflect.Complex64], 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(complexResult)
+		execDec("complex", instr, state, t, unsafe.Pointer(&data))
+		if data.a != 17+19i {
+			t.Errorf("complex a = %v not 17+19i", data.a)
+		}
+	}
+
+	// complex128
+	{
+		var data struct {
+			a complex128
+		}
+		instr := &decInstr{decOpMap[reflect.Complex128], 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(complexResult)
+		execDec("complex", instr, state, t, unsafe.Pointer(&data))
+		if data.a != 17+19i {
+			t.Errorf("complex a = %v not 17+19i", data.a)
+		}
+	}
+
+	// bytes == []uint8
+	{
+		var data struct {
+			a []byte
+		}
+		instr := &decInstr{decUint8Array, 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(bytesResult)
+		execDec("bytes", instr, state, t, unsafe.Pointer(&data))
+		if string(data.a) != "hello" {
+			t.Errorf(`bytes a = %q not "hello"`, string(data.a))
+		}
+	}
+
+	// string
+	{
+		var data struct {
+			a string
+		}
+		instr := &decInstr{decString, 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(bytesResult)
+		execDec("bytes", instr, state, t, unsafe.Pointer(&data))
+		if data.a != "hello" {
+			t.Errorf(`bytes a = %q not "hello"`, data.a)
+		}
+	}
+}
+
+func TestEndToEnd(t *testing.T) {
+	type T2 struct {
+		T string
+	}
+	s1 := "string1"
+	s2 := "string2"
+	type T1 struct {
+		A, B, C int
+		M       map[string]*float64
+		N       *[3]float64
+		Strs    *[2]string
+		Int64s  *[]int64
+		RI      complex64
+		S       string
+		Y       []byte
+		T       *T2
+	}
+	pi := 3.14159
+	e := 2.71828
+	t1 := &T1{
+		A:      17,
+		B:      18,
+		C:      -5,
+		M:      map[string]*float64{"pi": &pi, "e": &e},
+		N:      &[3]float64{1.5, 2.5, 3.5},
+		Strs:   &[2]string{s1, s2},
+		Int64s: &[]int64{77, 89, 123412342134},
+		RI:     17 - 23i,
+		S:      "Now is the time",
+		Y:      []byte("hello, sailor"),
+		T:      &T2{"this is T2"},
+	}
+	b := new(bytes.Buffer)
+	err := NewEncoder(b).Encode(t1)
+	if err != nil {
+		t.Error("encode:", err)
+	}
+	var _t1 T1
+	err = NewDecoder(b).Decode(&_t1)
+	if err != nil {
+		t.Fatal("decode:", err)
+	}
+	if !reflect.DeepEqual(t1, &_t1) {
+		t.Errorf("encode expected %v got %v", *t1, _t1)
+	}
+}
+
+func TestOverflow(t *testing.T) {
+	type inputT struct {
+		Maxi int64
+		Mini int64
+		Maxu uint64
+		Maxf float64
+		Minf float64
+		Maxc complex128
+		Minc complex128
+	}
+	var it inputT
+	var err os.Error
+	b := new(bytes.Buffer)
+	enc := NewEncoder(b)
+	dec := NewDecoder(b)
+
+	// int8
+	b.Reset()
+	it = inputT{
+		Maxi: math.MaxInt8 + 1,
+	}
+	type outi8 struct {
+		Maxi int8
+		Mini int8
+	}
+	var o1 outi8
+	enc.Encode(it)
+	err = dec.Decode(&o1)
+	if err == nil || err.String() != `value for "Maxi" out of range` {
+		t.Error("wrong overflow error for int8:", err)
+	}
+	it = inputT{
+		Mini: math.MinInt8 - 1,
+	}
+	b.Reset()
+	enc.Encode(it)
+	err = dec.Decode(&o1)
+	if err == nil || err.String() != `value for "Mini" out of range` {
+		t.Error("wrong underflow error for int8:", err)
+	}
+
+	// int16
+	b.Reset()
+	it = inputT{
+		Maxi: math.MaxInt16 + 1,
+	}
+	type outi16 struct {
+		Maxi int16
+		Mini int16
+	}
+	var o2 outi16
+	enc.Encode(it)
+	err = dec.Decode(&o2)
+	if err == nil || err.String() != `value for "Maxi" out of range` {
+		t.Error("wrong overflow error for int16:", err)
+	}
+	it = inputT{
+		Mini: math.MinInt16 - 1,
+	}
+	b.Reset()
+	enc.Encode(it)
+	err = dec.Decode(&o2)
+	if err == nil || err.String() != `value for "Mini" out of range` {
+		t.Error("wrong underflow error for int16:", err)
+	}
+
+	// int32
+	b.Reset()
+	it = inputT{
+		Maxi: math.MaxInt32 + 1,
+	}
+	type outi32 struct {
+		Maxi int32
+		Mini int32
+	}
+	var o3 outi32
+	enc.Encode(it)
+	err = dec.Decode(&o3)
+	if err == nil || err.String() != `value for "Maxi" out of range` {
+		t.Error("wrong overflow error for int32:", err)
+	}
+	it = inputT{
+		Mini: math.MinInt32 - 1,
+	}
+	b.Reset()
+	enc.Encode(it)
+	err = dec.Decode(&o3)
+	if err == nil || err.String() != `value for "Mini" out of range` {
+		t.Error("wrong underflow error for int32:", err)
+	}
+
+	// uint8
+	b.Reset()
+	it = inputT{
+		Maxu: math.MaxUint8 + 1,
+	}
+	type outu8 struct {
+		Maxu uint8
+	}
+	var o4 outu8
+	enc.Encode(it)
+	err = dec.Decode(&o4)
+	if err == nil || err.String() != `value for "Maxu" out of range` {
+		t.Error("wrong overflow error for uint8:", err)
+	}
+
+	// uint16
+	b.Reset()
+	it = inputT{
+		Maxu: math.MaxUint16 + 1,
+	}
+	type outu16 struct {
+		Maxu uint16
+	}
+	var o5 outu16
+	enc.Encode(it)
+	err = dec.Decode(&o5)
+	if err == nil || err.String() != `value for "Maxu" out of range` {
+		t.Error("wrong overflow error for uint16:", err)
+	}
+
+	// uint32
+	b.Reset()
+	it = inputT{
+		Maxu: math.MaxUint32 + 1,
+	}
+	type outu32 struct {
+		Maxu uint32
+	}
+	var o6 outu32
+	enc.Encode(it)
+	err = dec.Decode(&o6)
+	if err == nil || err.String() != `value for "Maxu" out of range` {
+		t.Error("wrong overflow error for uint32:", err)
+	}
+
+	// float32
+	b.Reset()
+	it = inputT{
+		Maxf: math.MaxFloat32 * 2,
+	}
+	type outf32 struct {
+		Maxf float32
+		Minf float32
+	}
+	var o7 outf32
+	enc.Encode(it)
+	err = dec.Decode(&o7)
+	if err == nil || err.String() != `value for "Maxf" out of range` {
+		t.Error("wrong overflow error for float32:", err)
+	}
+
+	// complex64
+	b.Reset()
+	it = inputT{
+		Maxc: complex(math.MaxFloat32*2, math.MaxFloat32*2),
+	}
+	type outc64 struct {
+		Maxc complex64
+		Minc complex64
+	}
+	var o8 outc64
+	enc.Encode(it)
+	err = dec.Decode(&o8)
+	if err == nil || err.String() != `value for "Maxc" out of range` {
+		t.Error("wrong overflow error for complex64:", err)
+	}
+}
+
+
+func TestNesting(t *testing.T) {
+	type RT struct {
+		A    string
+		Next *RT
+	}
+	rt := new(RT)
+	rt.A = "level1"
+	rt.Next = new(RT)
+	rt.Next.A = "level2"
+	b := new(bytes.Buffer)
+	NewEncoder(b).Encode(rt)
+	var drt RT
+	dec := NewDecoder(b)
+	err := dec.Decode(&drt)
+	if err != nil {
+		t.Fatal("decoder error:", err)
+	}
+	if drt.A != rt.A {
+		t.Errorf("nesting: encode expected %v got %v", *rt, drt)
+	}
+	if drt.Next == nil {
+		t.Errorf("nesting: recursion failed")
+	}
+	if drt.Next.A != rt.Next.A {
+		t.Errorf("nesting: encode expected %v got %v", *rt.Next, *drt.Next)
+	}
+}
+
+// These three structures have the same data with different indirections
+type T0 struct {
+	A int
+	B int
+	C int
+	D int
+}
+type T1 struct {
+	A int
+	B *int
+	C **int
+	D ***int
+}
+type T2 struct {
+	A ***int
+	B **int
+	C *int
+	D int
+}
+
+func TestAutoIndirection(t *testing.T) {
+	// First transfer t1 into t0
+	var t1 T1
+	t1.A = 17
+	t1.B = new(int)
+	*t1.B = 177
+	t1.C = new(*int)
+	*t1.C = new(int)
+	**t1.C = 1777
+	t1.D = new(**int)
+	*t1.D = new(*int)
+	**t1.D = new(int)
+	***t1.D = 17777
+	b := new(bytes.Buffer)
+	enc := NewEncoder(b)
+	enc.Encode(t1)
+	dec := NewDecoder(b)
+	var t0 T0
+	dec.Decode(&t0)
+	if t0.A != 17 || t0.B != 177 || t0.C != 1777 || t0.D != 17777 {
+		t.Errorf("t1->t0: expected {17 177 1777 17777}; got %v", t0)
+	}
+
+	// Now transfer t2 into t0
+	var t2 T2
+	t2.D = 17777
+	t2.C = new(int)
+	*t2.C = 1777
+	t2.B = new(*int)
+	*t2.B = new(int)
+	**t2.B = 177
+	t2.A = new(**int)
+	*t2.A = new(*int)
+	**t2.A = new(int)
+	***t2.A = 17
+	b.Reset()
+	enc.Encode(t2)
+	t0 = T0{}
+	dec.Decode(&t0)
+	if t0.A != 17 || t0.B != 177 || t0.C != 1777 || t0.D != 17777 {
+		t.Errorf("t2->t0 expected {17 177 1777 17777}; got %v", t0)
+	}
+
+	// Now transfer t0 into t1
+	t0 = T0{17, 177, 1777, 17777}
+	b.Reset()
+	enc.Encode(t0)
+	t1 = T1{}
+	dec.Decode(&t1)
+	if t1.A != 17 || *t1.B != 177 || **t1.C != 1777 || ***t1.D != 17777 {
+		t.Errorf("t0->t1 expected {17 177 1777 17777}; got {%d %d %d %d}", t1.A, *t1.B, **t1.C, ***t1.D)
+	}
+
+	// Now transfer t0 into t2
+	b.Reset()
+	enc.Encode(t0)
+	t2 = T2{}
+	dec.Decode(&t2)
+	if ***t2.A != 17 || **t2.B != 177 || *t2.C != 1777 || t2.D != 17777 {
+		t.Errorf("t0->t2 expected {17 177 1777 17777}; got {%d %d %d %d}", ***t2.A, **t2.B, *t2.C, t2.D)
+	}
+
+	// Now do t2 again but without pre-allocated pointers.
+	b.Reset()
+	enc.Encode(t0)
+	***t2.A = 0
+	**t2.B = 0
+	*t2.C = 0
+	t2.D = 0
+	dec.Decode(&t2)
+	if ***t2.A != 17 || **t2.B != 177 || *t2.C != 1777 || t2.D != 17777 {
+		t.Errorf("t0->t2 expected {17 177 1777 17777}; got {%d %d %d %d}", ***t2.A, **t2.B, *t2.C, t2.D)
+	}
+}
+
+type RT0 struct {
+	A int
+	B string
+	C float64
+}
+type RT1 struct {
+	C      float64
+	B      string
+	A      int
+	NotSet string
+}
+
+func TestReorderedFields(t *testing.T) {
+	var rt0 RT0
+	rt0.A = 17
+	rt0.B = "hello"
+	rt0.C = 3.14159
+	b := new(bytes.Buffer)
+	NewEncoder(b).Encode(rt0)
+	dec := NewDecoder(b)
+	var rt1 RT1
+	// Wire type is RT0, local type is RT1.
+	err := dec.Decode(&rt1)
+	if err != nil {
+		t.Fatal("decode error:", err)
+	}
+	if rt0.A != rt1.A || rt0.B != rt1.B || rt0.C != rt1.C {
+		t.Errorf("rt1->rt0: expected %v; got %v", rt0, rt1)
+	}
+}
+
+// Like an RT0 but with fields we'll ignore on the decode side.
+type IT0 struct {
+	A        int64
+	B        string
+	Ignore_d []int
+	Ignore_e [3]float64
+	Ignore_f bool
+	Ignore_g string
+	Ignore_h []byte
+	Ignore_i *RT1
+	Ignore_m map[string]int
+	C        float64
+}
+
+func TestIgnoredFields(t *testing.T) {
+	var it0 IT0
+	it0.A = 17
+	it0.B = "hello"
+	it0.C = 3.14159
+	it0.Ignore_d = []int{1, 2, 3}
+	it0.Ignore_e[0] = 1.0
+	it0.Ignore_e[1] = 2.0
+	it0.Ignore_e[2] = 3.0
+	it0.Ignore_f = true
+	it0.Ignore_g = "pay no attention"
+	it0.Ignore_h = []byte("to the curtain")
+	it0.Ignore_i = &RT1{3.1, "hi", 7, "hello"}
+	it0.Ignore_m = map[string]int{"one": 1, "two": 2}
+
+	b := new(bytes.Buffer)
+	NewEncoder(b).Encode(it0)
+	dec := NewDecoder(b)
+	var rt1 RT1
+	// Wire type is IT0, local type is RT1.
+	err := dec.Decode(&rt1)
+	if err != nil {
+		t.Error("error: ", err)
+	}
+	if int(it0.A) != rt1.A || it0.B != rt1.B || it0.C != rt1.C {
+		t.Errorf("rt0->rt1: expected %v; got %v", it0, rt1)
+	}
+}
+
+type Bad0 struct {
+	ch chan int
+	c  float64
+}
+
+var nilEncoder *Encoder
+
+func TestInvalidField(t *testing.T) {
+	var bad0 Bad0
+	bad0.ch = make(chan int)
+	b := new(bytes.Buffer)
+	err := nilEncoder.encode(b, reflect.NewValue(&bad0))
+	if err == nil {
+		t.Error("expected error; got none")
+	} else if strings.Index(err.String(), "type") < 0 {
+		t.Error("expected type error; got", err)
+	}
+}
+
+type Indirect struct {
+	A ***[3]int
+	S ***[]int
+	M ****map[string]int
+}
+
+type Direct struct {
+	A [3]int
+	S []int
+	M map[string]int
+}
+
+func TestIndirectSliceMapArray(t *testing.T) {
+	// Marshal indirect, unmarshal to direct.
+	i := new(Indirect)
+	i.A = new(**[3]int)
+	*i.A = new(*[3]int)
+	**i.A = new([3]int)
+	***i.A = [3]int{1, 2, 3}
+	i.S = new(**[]int)
+	*i.S = new(*[]int)
+	**i.S = new([]int)
+	***i.S = []int{4, 5, 6}
+	i.M = new(***map[string]int)
+	*i.M = new(**map[string]int)
+	**i.M = new(*map[string]int)
+	***i.M = new(map[string]int)
+	****i.M = map[string]int{"one": 1, "two": 2, "three": 3}
+	b := new(bytes.Buffer)
+	NewEncoder(b).Encode(i)
+	dec := NewDecoder(b)
+	var d Direct
+	err := dec.Decode(&d)
+	if err != nil {
+		t.Error("error: ", err)
+	}
+	if len(d.A) != 3 || d.A[0] != 1 || d.A[1] != 2 || d.A[2] != 3 {
+		t.Errorf("indirect to direct: d.A is %v not %v", d.A, ***i.A)
+	}
+	if len(d.S) != 3 || d.S[0] != 4 || d.S[1] != 5 || d.S[2] != 6 {
+		t.Errorf("indirect to direct: d.S is %v not %v", d.S, ***i.S)
+	}
+	if len(d.M) != 3 || d.M["one"] != 1 || d.M["two"] != 2 || d.M["three"] != 3 {
+		t.Errorf("indirect to direct: d.M is %v not %v", d.M, ***i.M)
+	}
+	// Marshal direct, unmarshal to indirect.
+	d.A = [3]int{11, 22, 33}
+	d.S = []int{44, 55, 66}
+	d.M = map[string]int{"four": 4, "five": 5, "six": 6}
+	i = new(Indirect)
+	b.Reset()
+	NewEncoder(b).Encode(d)
+	dec = NewDecoder(b)
+	err = dec.Decode(&i)
+	if err != nil {
+		t.Fatal("error: ", err)
+	}
+	if len(***i.A) != 3 || (***i.A)[0] != 11 || (***i.A)[1] != 22 || (***i.A)[2] != 33 {
+		t.Errorf("direct to indirect: ***i.A is %v not %v", ***i.A, d.A)
+	}
+	if len(***i.S) != 3 || (***i.S)[0] != 44 || (***i.S)[1] != 55 || (***i.S)[2] != 66 {
+		t.Errorf("direct to indirect: ***i.S is %v not %v", ***i.S, ***i.S)
+	}
+	if len(****i.M) != 3 || (****i.M)["four"] != 4 || (****i.M)["five"] != 5 || (****i.M)["six"] != 6 {
+		t.Errorf("direct to indirect: ****i.M is %v not %v", ****i.M, d.M)
+	}
+}
+
+// An interface with several implementations
+type Squarer interface {
+	Square() int
+}
+
+type Int int
+
+func (i Int) Square() int {
+	return int(i * i)
+}
+
+type Float float64
+
+func (f Float) Square() int {
+	return int(f * f)
+}
+
+type Vector []int
+
+func (v Vector) Square() int {
+	sum := 0
+	for _, x := range v {
+		sum += x * x
+	}
+	return sum
+}
+
+type Point struct {
+	a, b int
+}
+
+func (p Point) Square() int {
+	return p.a*p.a + p.b*p.b
+}
+
+// A struct with interfaces in it.
+type InterfaceItem struct {
+	I             int
+	Sq1, Sq2, Sq3 Squarer
+	F             float64
+	Sq            []Squarer
+}
+
+// The same struct without interfaces
+type NoInterfaceItem struct {
+	I int
+	F float64
+}
+
+func TestInterface(t *testing.T) {
+	iVal := Int(3)
+	fVal := Float(5)
+	// Sending a Vector will require that the receiver define a type in the middle of
+	// receiving the value for item2.
+	vVal := Vector{1, 2, 3}
+	b := new(bytes.Buffer)
+	item1 := &InterfaceItem{1, iVal, fVal, vVal, 11.5, []Squarer{iVal, fVal, nil, vVal}}
+	// Register the types.
+	Register(Int(0))
+	Register(Float(0))
+	Register(Vector{})
+	err := NewEncoder(b).Encode(item1)
+	if err != nil {
+		t.Error("expected no encode error; got", err)
+	}
+
+	item2 := InterfaceItem{}
+	err = NewDecoder(b).Decode(&item2)
+	if err != nil {
+		t.Fatal("decode:", err)
+	}
+	if item2.I != item1.I {
+		t.Error("normal int did not decode correctly")
+	}
+	if item2.Sq1 == nil || item2.Sq1.Square() != iVal.Square() {
+		t.Error("Int did not decode correctly")
+	}
+	if item2.Sq2 == nil || item2.Sq2.Square() != fVal.Square() {
+		t.Error("Float did not decode correctly")
+	}
+	if item2.Sq3 == nil || item2.Sq3.Square() != vVal.Square() {
+		t.Error("Vector did not decode correctly")
+	}
+	if item2.F != item1.F {
+		t.Error("normal float did not decode correctly")
+	}
+	// Now check that we received a slice of Squarers correctly, including a nil element
+	if len(item1.Sq) != len(item2.Sq) {
+		t.Fatalf("[]Squarer length wrong: got %d; expected %d", len(item2.Sq), len(item1.Sq))
+	}
+	for i, v1 := range item1.Sq {
+		v2 := item2.Sq[i]
+		if v1 == nil || v2 == nil {
+			if v1 != nil || v2 != nil {
+				t.Errorf("item %d inconsistent nils", i)
+			}
+			continue
+			if v1.Square() != v2.Square() {
+				t.Errorf("item %d inconsistent values: %v %v", i, v1, v2)
+			}
+		}
+	}
+
+}
+
+// A struct with all basic types, stored in interfaces.
+type BasicInterfaceItem struct {
+	Int, Int8, Int16, Int32, Int64      interface{}
+	Uint, Uint8, Uint16, Uint32, Uint64 interface{}
+	Float32, Float64                    interface{}
+	Complex64, Complex128               interface{}
+	Bool                                interface{}
+	String                              interface{}
+	Bytes                               interface{}
+}
+
+func TestInterfaceBasic(t *testing.T) {
+	b := new(bytes.Buffer)
+	item1 := &BasicInterfaceItem{
+		int(1), int8(1), int16(1), int32(1), int64(1),
+		uint(1), uint8(1), uint16(1), uint32(1), uint64(1),
+		float32(1), 1.0,
+		complex64(0i), complex128(0i),
+		true,
+		"hello",
+		[]byte("sailor"),
+	}
+	err := NewEncoder(b).Encode(item1)
+	if err != nil {
+		t.Error("expected no encode error; got", err)
+	}
+
+	item2 := &BasicInterfaceItem{}
+	err = NewDecoder(b).Decode(&item2)
+	if err != nil {
+		t.Fatal("decode:", err)
+	}
+	if !reflect.DeepEqual(item1, item2) {
+		t.Errorf("encode expected %v got %v", item1, item2)
+	}
+	// Hand check a couple for correct types.
+	if v, ok := item2.Bool.(bool); !ok || !v {
+		t.Error("boolean should be true")
+	}
+	if v, ok := item2.String.(string); !ok || v != item1.String.(string) {
+		t.Errorf("string should be %v is %v", item1.String, v)
+	}
+}
+
+type String string
+
+type PtrInterfaceItem struct {
+	Str1 interface{} // basic
+	Str2 interface{} // derived
+}
+
+// We'll send pointers; should receive values.
+// Also check that we can register T but send *T.
+func TestInterfacePointer(t *testing.T) {
+	b := new(bytes.Buffer)
+	str1 := "howdy"
+	str2 := String("kiddo")
+	item1 := &PtrInterfaceItem{
+		&str1,
+		&str2,
+	}
+	// Register the type.
+	Register(str2)
+	err := NewEncoder(b).Encode(item1)
+	if err != nil {
+		t.Error("expected no encode error; got", err)
+	}
+
+	item2 := &PtrInterfaceItem{}
+	err = NewDecoder(b).Decode(&item2)
+	if err != nil {
+		t.Fatal("decode:", err)
+	}
+	// Hand test for correct types and values.
+	if v, ok := item2.Str1.(string); !ok || v != str1 {
+		t.Errorf("basic string failed: %q should be %q", v, str1)
+	}
+	if v, ok := item2.Str2.(String); !ok || v != str2 {
+		t.Errorf("derived type String failed: %q should be %q", v, str2)
+	}
+}
+
+func TestIgnoreInterface(t *testing.T) {
+	iVal := Int(3)
+	fVal := Float(5)
+	// Sending a Point will require that the receiver define a type in the middle of
+	// receiving the value for item2.
+	pVal := Point{2, 3}
+	b := new(bytes.Buffer)
+	item1 := &InterfaceItem{1, iVal, fVal, pVal, 11.5, nil}
+	// Register the types.
+	Register(Int(0))
+	Register(Float(0))
+	Register(Point{})
+	err := NewEncoder(b).Encode(item1)
+	if err != nil {
+		t.Error("expected no encode error; got", err)
+	}
+
+	item2 := NoInterfaceItem{}
+	err = NewDecoder(b).Decode(&item2)
+	if err != nil {
+		t.Fatal("decode:", err)
+	}
+	if item2.I != item1.I {
+		t.Error("normal int did not decode correctly")
+	}
+	if item2.F != item2.F {
+		t.Error("normal float did not decode correctly")
+	}
+}
+
+type U struct {
+	A int
+	B string
+	c float64
+	D uint
+}
+
+func TestUnexportedFields(t *testing.T) {
+	var u0 U
+	u0.A = 17
+	u0.B = "hello"
+	u0.c = 3.14159
+	u0.D = 23
+	b := new(bytes.Buffer)
+	NewEncoder(b).Encode(u0)
+	dec := NewDecoder(b)
+	var u1 U
+	u1.c = 1234.
+	err := dec.Decode(&u1)
+	if err != nil {
+		t.Fatal("decode error:", err)
+	}
+	if u0.A != u0.A || u0.B != u1.B || u0.D != u1.D {
+		t.Errorf("u1->u0: expected %v; got %v", u0, u1)
+	}
+	if u1.c != 1234. {
+		t.Error("u1.c modified")
+	}
+}
+
+// A type that won't be defined in the gob until we send it in an interface value.
+type OnTheFly struct {
+	A int
+}
+
+type DT struct {
+	//	X OnTheFly
+	A     int
+	B     string
+	C     float64
+	I     interface{}
+	J     interface{}
+	I_nil interface{}
+	M     map[string]int
+	T     [3]int
+	S     []string
+}
+
+func TestDebug(t *testing.T) {
+	if debugFunc == nil {
+		return
+	}
+	Register(OnTheFly{})
+	var dt DT
+	dt.A = 17
+	dt.B = "hello"
+	dt.C = 3.14159
+	dt.I = 271828
+	dt.J = OnTheFly{3}
+	dt.I_nil = nil
+	dt.M = map[string]int{"one": 1, "two": 2}
+	dt.T = [3]int{11, 22, 33}
+	dt.S = []string{"hi", "joe"}
+	b := new(bytes.Buffer)
+	err := NewEncoder(b).Encode(dt)
+	if err != nil {
+		t.Fatal("encode:", err)
+	}
+	debugBuffer := bytes.NewBuffer(b.Bytes())
+	dt2 := &DT{}
+	err = NewDecoder(b).Decode(&dt2)
+	if err != nil {
+		t.Error("decode:", err)
+	}
+	debugFunc(debugBuffer)
+}
diff --git a/libgo/go/gob/decode.go b/libgo/go/gob/decode.go
new file mode 100644
index 000000000..2db75215c
--- /dev/null
+++ b/libgo/go/gob/decode.go
@@ -0,0 +1,1020 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package gob
+
+// TODO(rsc): When garbage collector changes, revisit
+// the allocations in this file that use unsafe.Pointer.
+
+import (
+	"bytes"
+	"io"
+	"math"
+	"os"
+	"reflect"
+	"unicode"
+	"unsafe"
+	"utf8"
+)
+
+var (
+	errBadUint = os.ErrorString("gob: encoded unsigned integer out of range")
+	errBadType = os.ErrorString("gob: unknown type id or corrupted data")
+	errRange   = os.ErrorString("gob: internal error: field numbers out of bounds")
+)
+
+// The execution state of an instance of the decoder. A new state
+// is created for nested objects.
+type decodeState struct {
+	dec *Decoder
+	// The buffer is stored with an extra indirection because it may be replaced
+	// if we load a type during decode (when reading an interface value).
+	b        **bytes.Buffer
+	fieldnum int // the last field number read.
+	buf      []byte
+}
+
+func newDecodeState(dec *Decoder, b **bytes.Buffer) *decodeState {
+	d := new(decodeState)
+	d.dec = dec
+	d.b = b
+	d.buf = make([]byte, uint64Size)
+	return d
+}
+
+func overflow(name string) os.ErrorString {
+	return os.ErrorString(`value for "` + name + `" out of range`)
+}
+
+// decodeUintReader reads an encoded unsigned integer from an io.Reader.
+// Used only by the Decoder to read the message length.
+func decodeUintReader(r io.Reader, buf []byte) (x uint64, err os.Error) {
+	_, err = r.Read(buf[0:1])
+	if err != nil {
+		return
+	}
+	b := buf[0]
+	if b <= 0x7f {
+		return uint64(b), nil
+	}
+	nb := -int(int8(b))
+	if nb > uint64Size {
+		err = errBadUint
+		return
+	}
+	var n int
+	n, err = io.ReadFull(r, buf[0:nb])
+	if err != nil {
+		if err == os.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+		return
+	}
+	// Could check that the high byte is zero but it's not worth it.
+	for i := 0; i < n; i++ {
+		x <<= 8
+		x |= uint64(buf[i])
+	}
+	return
+}
+
+// decodeUint reads an encoded unsigned integer from state.r.
+// Does not check for overflow.
+func (state *decodeState) decodeUint() (x uint64) {
+	b, err := state.b.ReadByte()
+	if err != nil {
+		error(err)
+	}
+	if b <= 0x7f {
+		return uint64(b)
+	}
+	nb := -int(int8(b))
+	if nb > uint64Size {
+		error(errBadUint)
+	}
+	n, err := state.b.Read(state.buf[0:nb])
+	if err != nil {
+		error(err)
+	}
+	// Don't need to check error; it's safe to loop regardless.
+	// Could check that the high byte is zero but it's not worth it.
+	for i := 0; i < n; i++ {
+		x <<= 8
+		x |= uint64(state.buf[i])
+	}
+	return x
+}
+
+// decodeInt reads an encoded signed integer from state.r.
+// Does not check for overflow.
+func (state *decodeState) decodeInt() int64 {
+	x := state.decodeUint()
+	if x&1 != 0 {
+		return ^int64(x >> 1)
+	}
+	return int64(x >> 1)
+}
+
+type decOp func(i *decInstr, state *decodeState, p unsafe.Pointer)
+
+// The 'instructions' of the decoding machine
+type decInstr struct {
+	op     decOp
+	field  int            // field number of the wire type
+	indir  int            // how many pointer indirections to reach the value in the struct
+	offset uintptr        // offset in the structure of the field to encode
+	ovfl   os.ErrorString // error message for overflow/underflow (for arrays, of the elements)
+}
+
+// Since the encoder writes no zeros, if we arrive at a decoder we have
+// a value to extract and store.  The field number has already been read
+// (it's how we knew to call this decoder).
+// Each decoder is responsible for handling any indirections associated
+// with the data structure.  If any pointer so reached is nil, allocation must
+// be done.
+
+// Walk the pointer hierarchy, allocating if we find a nil.  Stop one before the end.
+func decIndirect(p unsafe.Pointer, indir int) unsafe.Pointer {
+	for ; indir > 1; indir-- {
+		if *(*unsafe.Pointer)(p) == nil {
+			// Allocation required
+			*(*unsafe.Pointer)(p) = unsafe.Pointer(new(unsafe.Pointer))
+		}
+		p = *(*unsafe.Pointer)(p)
+	}
+	return p
+}
+
+func ignoreUint(i *decInstr, state *decodeState, p unsafe.Pointer) {
+	state.decodeUint()
+}
+
+func ignoreTwoUints(i *decInstr, state *decodeState, p unsafe.Pointer) {
+	state.decodeUint()
+	state.decodeUint()
+}
+
+func decBool(i *decInstr, state *decodeState, p unsafe.Pointer) {
+	if i.indir > 0 {
+		if *(*unsafe.Pointer)(p) == nil {
+			*(*unsafe.Pointer)(p) = unsafe.Pointer(new(bool))
+		}
+		p = *(*unsafe.Pointer)(p)
+	}
+	*(*bool)(p) = state.decodeInt() != 0
+}
+
+func decInt8(i *decInstr, state *decodeState, p unsafe.Pointer) {
+	if i.indir > 0 {
+		if *(*unsafe.Pointer)(p) == nil {
+			*(*unsafe.Pointer)(p) = unsafe.Pointer(new(int8))
+		}
+		p = *(*unsafe.Pointer)(p)
+	}
+	v := state.decodeInt()
+	if v < math.MinInt8 || math.MaxInt8 < v {
+		error(i.ovfl)
+	} else {
+		*(*int8)(p) = int8(v)
+	}
+}
+
+func decUint8(i *decInstr, state *decodeState, p unsafe.Pointer) {
+	if i.indir > 0 {
+		if *(*unsafe.Pointer)(p) == nil {
+			*(*unsafe.Pointer)(p) = unsafe.Pointer(new(uint8))
+		}
+		p = *(*unsafe.Pointer)(p)
+	}
+	v := state.decodeUint()
+	if math.MaxUint8 < v {
+		error(i.ovfl)
+	} else {
+		*(*uint8)(p) = uint8(v)
+	}
+}
+
+func decInt16(i *decInstr, state *decodeState, p unsafe.Pointer) {
+	if i.indir > 0 {
+		if *(*unsafe.Pointer)(p) == nil {
+			*(*unsafe.Pointer)(p) = unsafe.Pointer(new(int16))
+		}
+		p = *(*unsafe.Pointer)(p)
+	}
+	v := state.decodeInt()
+	if v < math.MinInt16 || math.MaxInt16 < v {
+		error(i.ovfl)
+	} else {
+		*(*int16)(p) = int16(v)
+	}
+}
+
+func decUint16(i *decInstr, state *decodeState, p unsafe.Pointer) {
+	if i.indir > 0 {
+		if *(*unsafe.Pointer)(p) == nil {
+			*(*unsafe.Pointer)(p) = unsafe.Pointer(new(uint16))
+		}
+		p = *(*unsafe.Pointer)(p)
+	}
+	v := state.decodeUint()
+	if math.MaxUint16 < v {
+		error(i.ovfl)
+	} else {
+		*(*uint16)(p) = uint16(v)
+	}
+}
+
+func decInt32(i *decInstr, state *decodeState, p unsafe.Pointer) {
+	if i.indir > 0 {
+		if *(*unsafe.Pointer)(p) == nil {
+			*(*unsafe.Pointer)(p) = unsafe.Pointer(new(int32))
+		}
+		p = *(*unsafe.Pointer)(p)
+	}
+	v := state.decodeInt()
+	if v < math.MinInt32 || math.MaxInt32 < v {
+		error(i.ovfl)
+	} else {
+		*(*int32)(p) = int32(v)
+	}
+}
+
+func decUint32(i *decInstr, state *decodeState, p unsafe.Pointer) {
+	if i.indir > 0 {
+		if *(*unsafe.Pointer)(p) == nil {
+			*(*unsafe.Pointer)(p) = unsafe.Pointer(new(uint32))
+		}
+		p = *(*unsafe.Pointer)(p)
+	}
+	v := state.decodeUint()
+	if math.MaxUint32 < v {
+		error(i.ovfl)
+	} else {
+		*(*uint32)(p) = uint32(v)
+	}
+}
+
+func decInt64(i *decInstr, state *decodeState, p unsafe.Pointer) {
+	if i.indir > 0 {
+		if *(*unsafe.Pointer)(p) == nil {
+			*(*unsafe.Pointer)(p) = unsafe.Pointer(new(int64))
+		}
+		p = *(*unsafe.Pointer)(p)
+	}
+	*(*int64)(p) = int64(state.decodeInt())
+}
+
+func decUint64(i *decInstr, state *decodeState, p unsafe.Pointer) {
+	if i.indir > 0 {
+		if *(*unsafe.Pointer)(p) == nil {
+			*(*unsafe.Pointer)(p) = unsafe.Pointer(new(uint64))
+		}
+		p = *(*unsafe.Pointer)(p)
+	}
+	*(*uint64)(p) = uint64(state.decodeUint())
+}
+
+// Floating-point numbers are transmitted as uint64s holding the bits
+// of the underlying representation.  They are sent byte-reversed, with
+// the exponent end coming out first, so integer floating point numbers
+// (for example) transmit more compactly.  This routine does the
+// unswizzling.
+func floatFromBits(u uint64) float64 {
+	var v uint64
+	for i := 0; i < 8; i++ {
+		v <<= 8
+		v |= u & 0xFF
+		u >>= 8
+	}
+	return math.Float64frombits(v)
+}
+
+func storeFloat32(i *decInstr, state *decodeState, p unsafe.Pointer) {
+	v := floatFromBits(state.decodeUint())
+	av := v
+	if av < 0 {
+		av = -av
+	}
+	// +Inf is OK in both 32- and 64-bit floats.  Underflow is always OK.
+	if math.MaxFloat32 < av && av <= math.MaxFloat64 {
+		error(i.ovfl)
+	} else {
+		*(*float32)(p) = float32(v)
+	}
+}
+
+func decFloat32(i *decInstr, state *decodeState, p unsafe.Pointer) {
+	if i.indir > 0 {
+		if *(*unsafe.Pointer)(p) == nil {
+			*(*unsafe.Pointer)(p) = unsafe.Pointer(new(float32))
+		}
+		p = *(*unsafe.Pointer)(p)
+	}
+	storeFloat32(i, state, p)
+}
+
+func decFloat64(i *decInstr, state *decodeState, p unsafe.Pointer) {
+	if i.indir > 0 {
+		if *(*unsafe.Pointer)(p) == nil {
+			*(*unsafe.Pointer)(p) = unsafe.Pointer(new(float64))
+		}
+		p = *(*unsafe.Pointer)(p)
+	}
+	*(*float64)(p) = floatFromBits(uint64(state.decodeUint()))
+}
+
+// Complex numbers are just a pair of floating-point numbers, real part first.
+func decComplex64(i *decInstr, state *decodeState, p unsafe.Pointer) {
+	if i.indir > 0 {
+		if *(*unsafe.Pointer)(p) == nil {
+			*(*unsafe.Pointer)(p) = unsafe.Pointer(new(complex64))
+		}
+		p = *(*unsafe.Pointer)(p)
+	}
+	storeFloat32(i, state, p)
+	storeFloat32(i, state, unsafe.Pointer(uintptr(p)+uintptr(unsafe.Sizeof(float32(0)))))
+}
+
+func decComplex128(i *decInstr, state *decodeState, p unsafe.Pointer) {
+	if i.indir > 0 {
+		if *(*unsafe.Pointer)(p) == nil {
+			*(*unsafe.Pointer)(p) = unsafe.Pointer(new(complex128))
+		}
+		p = *(*unsafe.Pointer)(p)
+	}
+	real := floatFromBits(uint64(state.decodeUint()))
+	imag := floatFromBits(uint64(state.decodeUint()))
+	*(*complex128)(p) = complex(real, imag)
+}
+
+// uint8 arrays are encoded as an unsigned count followed by the raw bytes.
+func decUint8Array(i *decInstr, state *decodeState, p unsafe.Pointer) {
+	if i.indir > 0 {
+		if *(*unsafe.Pointer)(p) == nil {
+			*(*unsafe.Pointer)(p) = unsafe.Pointer(new([]uint8))
+		}
+		p = *(*unsafe.Pointer)(p)
+	}
+	b := make([]uint8, state.decodeUint())
+	state.b.Read(b)
+	*(*[]uint8)(p) = b
+}
+
+// Strings are encoded as an unsigned count followed by the raw bytes.
+func decString(i *decInstr, state *decodeState, p unsafe.Pointer) {
+	if i.indir > 0 {
+		if *(*unsafe.Pointer)(p) == nil {
+			*(*unsafe.Pointer)(p) = unsafe.Pointer(new([]byte))
+		}
+		p = *(*unsafe.Pointer)(p)
+	}
+	b := make([]byte, state.decodeUint())
+	state.b.Read(b)
+	*(*string)(p) = string(b)
+}
+
+func ignoreUint8Array(i *decInstr, state *decodeState, p unsafe.Pointer) {
+	b := make([]byte, state.decodeUint())
+	state.b.Read(b)
+}
+
+// Execution engine
+
+// The encoder engine is an array of instructions indexed by field number of the incoming
+// decoder.  It is executed with random access according to field number.
+type decEngine struct {
+	instr    []decInstr
+	numInstr int // the number of active instructions
+}
+
+// allocate makes sure storage is available for an object of underlying type rtyp
+// that is indir levels of indirection through p.
+func allocate(rtyp reflect.Type, p uintptr, indir int) uintptr {
+	if indir == 0 {
+		return p
+	}
+	up := unsafe.Pointer(p)
+	if indir > 1 {
+		up = decIndirect(up, indir)
+	}
+	if *(*unsafe.Pointer)(up) == nil {
+		// Allocate object.
+		*(*unsafe.Pointer)(up) = unsafe.New(rtyp)
+	}
+	return *(*uintptr)(up)
+}
+
+func (dec *Decoder) decodeSingle(engine *decEngine, rtyp reflect.Type, b **bytes.Buffer, p uintptr, indir int) (err os.Error) {
+	defer catchError(&err)
+	p = allocate(rtyp, p, indir)
+	state := newDecodeState(dec, b)
+	state.fieldnum = singletonField
+	basep := p
+	delta := int(state.decodeUint())
+	if delta != 0 {
+		errorf("gob decode: corrupted data: non-zero delta for singleton")
+	}
+	instr := &engine.instr[singletonField]
+	ptr := unsafe.Pointer(basep) // offset will be zero
+	if instr.indir > 1 {
+		ptr = decIndirect(ptr, instr.indir)
+	}
+	instr.op(instr, state, ptr)
+	return nil
+}
+
+func (dec *Decoder) decodeStruct(engine *decEngine, rtyp *reflect.StructType, b **bytes.Buffer, p uintptr, indir int) (err os.Error) {
+	defer catchError(&err)
+	p = allocate(rtyp, p, indir)
+	state := newDecodeState(dec, b)
+	state.fieldnum = -1
+	basep := p
+	for state.b.Len() > 0 {
+		delta := int(state.decodeUint())
+		if delta < 0 {
+			errorf("gob decode: corrupted data: negative delta")
+		}
+		if delta == 0 { // struct terminator is zero delta fieldnum
+			break
+		}
+		fieldnum := state.fieldnum + delta
+		if fieldnum >= len(engine.instr) {
+			error(errRange)
+			break
+		}
+		instr := &engine.instr[fieldnum]
+		p := unsafe.Pointer(basep + instr.offset)
+		if instr.indir > 1 {
+			p = decIndirect(p, instr.indir)
+		}
+		instr.op(instr, state, p)
+		state.fieldnum = fieldnum
+	}
+	return nil
+}
+
+func (dec *Decoder) ignoreStruct(engine *decEngine, b **bytes.Buffer) (err os.Error) {
+	defer catchError(&err)
+	state := newDecodeState(dec, b)
+	state.fieldnum = -1
+	for state.b.Len() > 0 {
+		delta := int(state.decodeUint())
+		if delta < 0 {
+			errorf("gob ignore decode: corrupted data: negative delta")
+		}
+		if delta == 0 { // struct terminator is zero delta fieldnum
+			break
+		}
+		fieldnum := state.fieldnum + delta
+		if fieldnum >= len(engine.instr) {
+			error(errRange)
+		}
+		instr := &engine.instr[fieldnum]
+		instr.op(instr, state, unsafe.Pointer(nil))
+		state.fieldnum = fieldnum
+	}
+	return nil
+}
+
+func (dec *Decoder) decodeArrayHelper(state *decodeState, p uintptr, elemOp decOp, elemWid uintptr, length, elemIndir int, ovfl os.ErrorString) {
+	instr := &decInstr{elemOp, 0, elemIndir, 0, ovfl}
+	for i := 0; i < length; i++ {
+		up := unsafe.Pointer(p)
+		if elemIndir > 1 {
+			up = decIndirect(up, elemIndir)
+		}
+		elemOp(instr, state, up)
+		p += uintptr(elemWid)
+	}
+}
+
+func (dec *Decoder) decodeArray(atyp *reflect.ArrayType, state *decodeState, p uintptr, elemOp decOp, elemWid uintptr, length, indir, elemIndir int, ovfl os.ErrorString) {
+	if indir > 0 {
+		p = allocate(atyp, p, 1) // All but the last level has been allocated by dec.Indirect
+	}
+	if n := state.decodeUint(); n != uint64(length) {
+		errorf("gob: length mismatch in decodeArray")
+	}
+	dec.decodeArrayHelper(state, p, elemOp, elemWid, length, elemIndir, ovfl)
+}
+
+func decodeIntoValue(state *decodeState, op decOp, indir int, v reflect.Value, ovfl os.ErrorString) reflect.Value {
+	instr := &decInstr{op, 0, indir, 0, ovfl}
+	up := unsafe.Pointer(v.Addr())
+	if indir > 1 {
+		up = decIndirect(up, indir)
+	}
+	op(instr, state, up)
+	return v
+}
+
+func (dec *Decoder) decodeMap(mtyp *reflect.MapType, state *decodeState, p uintptr, keyOp, elemOp decOp, indir, keyIndir, elemIndir int, ovfl os.ErrorString) {
+	if indir > 0 {
+		p = allocate(mtyp, p, 1) // All but the last level has been allocated by dec.Indirect
+	}
+	up := unsafe.Pointer(p)
+	if *(*unsafe.Pointer)(up) == nil { // maps are represented as a pointer in the runtime
+		// Allocate map.
+		*(*unsafe.Pointer)(up) = unsafe.Pointer(reflect.MakeMap(mtyp).Get())
+	}
+	// Maps cannot be accessed by moving addresses around the way
+	// that slices etc. can.  We must recover a full reflection value for
+	// the iteration.
+	v := reflect.NewValue(unsafe.Unreflect(mtyp, unsafe.Pointer((p)))).(*reflect.MapValue)
+	n := int(state.decodeUint())
+	for i := 0; i < n; i++ {
+		key := decodeIntoValue(state, keyOp, keyIndir, reflect.MakeZero(mtyp.Key()), ovfl)
+		elem := decodeIntoValue(state, elemOp, elemIndir, reflect.MakeZero(mtyp.Elem()), ovfl)
+		v.SetElem(key, elem)
+	}
+}
+
+func (dec *Decoder) ignoreArrayHelper(state *decodeState, elemOp decOp, length int) {
+	instr := &decInstr{elemOp, 0, 0, 0, os.ErrorString("no error")}
+	for i := 0; i < length; i++ {
+		elemOp(instr, state, nil)
+	}
+}
+
+func (dec *Decoder) ignoreArray(state *decodeState, elemOp decOp, length int) {
+	if n := state.decodeUint(); n != uint64(length) {
+		errorf("gob: length mismatch in ignoreArray")
+	}
+	dec.ignoreArrayHelper(state, elemOp, length)
+}
+
+func (dec *Decoder) ignoreMap(state *decodeState, keyOp, elemOp decOp) {
+	n := int(state.decodeUint())
+	keyInstr := &decInstr{keyOp, 0, 0, 0, os.ErrorString("no error")}
+	elemInstr := &decInstr{elemOp, 0, 0, 0, os.ErrorString("no error")}
+	for i := 0; i < n; i++ {
+		keyOp(keyInstr, state, nil)
+		elemOp(elemInstr, state, nil)
+	}
+}
+
+func (dec *Decoder) decodeSlice(atyp *reflect.SliceType, state *decodeState, p uintptr, elemOp decOp, elemWid uintptr, indir, elemIndir int, ovfl os.ErrorString) {
+	n := int(uintptr(state.decodeUint()))
+	if indir > 0 {
+		up := unsafe.Pointer(p)
+		if *(*unsafe.Pointer)(up) == nil {
+			// Allocate the slice header.
+			*(*unsafe.Pointer)(up) = unsafe.Pointer(new([]unsafe.Pointer))
+		}
+		p = *(*uintptr)(up)
+	}
+	// Allocate storage for the slice elements, that is, the underlying array.
+	// Always write a header at p.
+	hdrp := (*reflect.SliceHeader)(unsafe.Pointer(p))
+	hdrp.Data = uintptr(unsafe.NewArray(atyp.Elem(), n))
+	hdrp.Len = n
+	hdrp.Cap = n
+	dec.decodeArrayHelper(state, hdrp.Data, elemOp, elemWid, n, elemIndir, ovfl)
+}
+
+func (dec *Decoder) ignoreSlice(state *decodeState, elemOp decOp) {
+	dec.ignoreArrayHelper(state, elemOp, int(state.decodeUint()))
+}
+
+// setInterfaceValue sets an interface value to a concrete value through
+// reflection.  If the concrete value does not implement the interface, the
+// setting will panic.  This routine turns the panic into an error return.
+// This dance avoids manually checking that the value satisfies the
+// interface.
+// TODO(rsc): avoid panic+recover after fixing issue 327.
+func setInterfaceValue(ivalue *reflect.InterfaceValue, value reflect.Value) {
+	defer func() {
+		if e := recover(); e != nil {
+			error(e.(os.Error))
+		}
+	}()
+	ivalue.Set(value)
+}
+
+// decodeInterface receives the name of a concrete type followed by its value.
+// If the name is empty, the value is nil and no value is sent.
+func (dec *Decoder) decodeInterface(ityp *reflect.InterfaceType, state *decodeState, p uintptr, indir int) {
+	// Create an interface reflect.Value.  We need one even for the nil case.
+	ivalue := reflect.MakeZero(ityp).(*reflect.InterfaceValue)
+	// Read the name of the concrete type.
+	b := make([]byte, state.decodeUint())
+	state.b.Read(b)
+	name := string(b)
+	if name == "" {
+		// Copy the representation of the nil interface value to the target.
+		// This is horribly unsafe and special.
+		*(*[2]uintptr)(unsafe.Pointer(p)) = ivalue.Get()
+		return
+	}
+	// The concrete type must be registered.
+	typ, ok := nameToConcreteType[name]
+	if !ok {
+		errorf("gob: name not registered for interface: %q", name)
+	}
+	// Read the concrete value.
+	value := reflect.MakeZero(typ)
+	dec.decodeValueFromBuffer(value, false, true)
+	if dec.err != nil {
+		error(dec.err)
+	}
+	// Allocate the destination interface value.
+	if indir > 0 {
+		p = allocate(ityp, p, 1) // All but the last level has been allocated by dec.Indirect
+	}
+	// Assign the concrete value to the interface.
+	// Tread carefully; it might not satisfy the interface.
+	setInterfaceValue(ivalue, value)
+	// Copy the representation of the interface value to the target.
+	// This is horribly unsafe and special.
+	*(*[2]uintptr)(unsafe.Pointer(p)) = ivalue.Get()
+}
+
+func (dec *Decoder) ignoreInterface(state *decodeState) {
+	// Read the name of the concrete type.
+	b := make([]byte, state.decodeUint())
+	_, err := state.b.Read(b)
+	if err != nil {
+		error(err)
+	}
+	dec.decodeValueFromBuffer(nil, true, true)
+	if dec.err != nil {
+		error(err)
+	}
+}
+
+// Index by Go types.
+var decOpMap = []decOp{
+	reflect.Bool:       decBool,
+	reflect.Int8:       decInt8,
+	reflect.Int16:      decInt16,
+	reflect.Int32:      decInt32,
+	reflect.Int64:      decInt64,
+	reflect.Uint8:      decUint8,
+	reflect.Uint16:     decUint16,
+	reflect.Uint32:     decUint32,
+	reflect.Uint64:     decUint64,
+	reflect.Float32:    decFloat32,
+	reflect.Float64:    decFloat64,
+	reflect.Complex64:  decComplex64,
+	reflect.Complex128: decComplex128,
+	reflect.String:     decString,
+}
+
+// Indexed by gob types.  tComplex will be added during type.init().
+var decIgnoreOpMap = map[typeId]decOp{
+	tBool:    ignoreUint,
+	tInt:     ignoreUint,
+	tUint:    ignoreUint,
+	tFloat:   ignoreUint,
+	tBytes:   ignoreUint8Array,
+	tString:  ignoreUint8Array,
+	tComplex: ignoreTwoUints,
+}
+
+// Return the decoding op for the base type under rt and
+// the indirection count to reach it.
+func (dec *Decoder) decOpFor(wireId typeId, rt reflect.Type, name string) (decOp, int) {
+	typ, indir := indirect(rt)
+	var op decOp
+	k := typ.Kind()
+	if int(k) < len(decOpMap) {
+		op = decOpMap[k]
+	}
+	if op == nil {
+		// Special cases
+		switch t := typ.(type) {
+		case *reflect.ArrayType:
+			name = "element of " + name
+			elemId := dec.wireType[wireId].ArrayT.Elem
+			elemOp, elemIndir := dec.decOpFor(elemId, t.Elem(), name)
+			ovfl := overflow(name)
+			op = func(i *decInstr, state *decodeState, p unsafe.Pointer) {
+				state.dec.decodeArray(t, state, uintptr(p), elemOp, t.Elem().Size(), t.Len(), i.indir, elemIndir, ovfl)
+			}
+
+		case *reflect.MapType:
+			name = "element of " + name
+			keyId := dec.wireType[wireId].MapT.Key
+			elemId := dec.wireType[wireId].MapT.Elem
+			keyOp, keyIndir := dec.decOpFor(keyId, t.Key(), name)
+			elemOp, elemIndir := dec.decOpFor(elemId, t.Elem(), name)
+			ovfl := overflow(name)
+			op = func(i *decInstr, state *decodeState, p unsafe.Pointer) {
+				up := unsafe.Pointer(p)
+				state.dec.decodeMap(t, state, uintptr(up), keyOp, elemOp, i.indir, keyIndir, elemIndir, ovfl)
+			}
+
+		case *reflect.SliceType:
+			name = "element of " + name
+			if t.Elem().Kind() == reflect.Uint8 {
+				op = decUint8Array
+				break
+			}
+			var elemId typeId
+			if tt, ok := builtinIdToType[wireId]; ok {
+				elemId = tt.(*sliceType).Elem
+			} else {
+				elemId = dec.wireType[wireId].SliceT.Elem
+			}
+			elemOp, elemIndir := dec.decOpFor(elemId, t.Elem(), name)
+			ovfl := overflow(name)
+			op = func(i *decInstr, state *decodeState, p unsafe.Pointer) {
+				state.dec.decodeSlice(t, state, uintptr(p), elemOp, t.Elem().Size(), i.indir, elemIndir, ovfl)
+			}
+
+		case *reflect.StructType:
+			// Generate a closure that calls out to the engine for the nested type.
+			enginePtr, err := dec.getDecEnginePtr(wireId, typ)
+			if err != nil {
+				error(err)
+			}
+			op = func(i *decInstr, state *decodeState, p unsafe.Pointer) {
+				// indirect through enginePtr to delay evaluation for recursive structs
+				err = dec.decodeStruct(*enginePtr, t, state.b, uintptr(p), i.indir)
+				if err != nil {
+					error(err)
+				}
+			}
+		case *reflect.InterfaceType:
+			op = func(i *decInstr, state *decodeState, p unsafe.Pointer) {
+				dec.decodeInterface(t, state, uintptr(p), i.indir)
+			}
+		}
+	}
+	if op == nil {
+		errorf("gob: decode can't handle type %s", rt.String())
+	}
+	return op, indir
+}
+
+// Return the decoding op for a field that has no destination.
+func (dec *Decoder) decIgnoreOpFor(wireId typeId) decOp {
+	op, ok := decIgnoreOpMap[wireId]
+	if !ok {
+		if wireId == tInterface {
+			// Special case because it's a method: the ignored item might
+			// define types and we need to record their state in the decoder.
+			op = func(i *decInstr, state *decodeState, p unsafe.Pointer) {
+				dec.ignoreInterface(state)
+			}
+			return op
+		}
+		// Special cases
+		wire := dec.wireType[wireId]
+		switch {
+		case wire == nil:
+			panic("internal error: can't find ignore op for type " + wireId.string())
+		case wire.ArrayT != nil:
+			elemId := wire.ArrayT.Elem
+			elemOp := dec.decIgnoreOpFor(elemId)
+			op = func(i *decInstr, state *decodeState, p unsafe.Pointer) {
+				state.dec.ignoreArray(state, elemOp, wire.ArrayT.Len)
+			}
+
+		case wire.MapT != nil:
+			keyId := dec.wireType[wireId].MapT.Key
+			elemId := dec.wireType[wireId].MapT.Elem
+			keyOp := dec.decIgnoreOpFor(keyId)
+			elemOp := dec.decIgnoreOpFor(elemId)
+			op = func(i *decInstr, state *decodeState, p unsafe.Pointer) {
+				state.dec.ignoreMap(state, keyOp, elemOp)
+			}
+
+		case wire.SliceT != nil:
+			elemId := wire.SliceT.Elem
+			elemOp := dec.decIgnoreOpFor(elemId)
+			op = func(i *decInstr, state *decodeState, p unsafe.Pointer) {
+				state.dec.ignoreSlice(state, elemOp)
+			}
+
+		case wire.StructT != nil:
+			// Generate a closure that calls out to the engine for the nested type.
+			enginePtr, err := dec.getIgnoreEnginePtr(wireId)
+			if err != nil {
+				error(err)
+			}
+			op = func(i *decInstr, state *decodeState, p unsafe.Pointer) {
+				// indirect through enginePtr to delay evaluation for recursive structs
+				state.dec.ignoreStruct(*enginePtr, state.b)
+			}
+		}
+	}
+	if op == nil {
+		errorf("ignore can't handle type %s", wireId.string())
+	}
+	return op
+}
+
+// Are these two gob Types compatible?
+// Answers the question for basic types, arrays, and slices.
+// Structs are considered ok; fields will be checked later.
+func (dec *Decoder) compatibleType(fr reflect.Type, fw typeId) bool {
+	fr, _ = indirect(fr)
+	switch t := fr.(type) {
+	default:
+		// map, chan, etc: cannot handle.
+		return false
+	case *reflect.BoolType:
+		return fw == tBool
+	case *reflect.IntType:
+		return fw == tInt
+	case *reflect.UintType:
+		return fw == tUint
+	case *reflect.FloatType:
+		return fw == tFloat
+	case *reflect.ComplexType:
+		return fw == tComplex
+	case *reflect.StringType:
+		return fw == tString
+	case *reflect.InterfaceType:
+		return fw == tInterface
+	case *reflect.ArrayType:
+		wire, ok := dec.wireType[fw]
+		if !ok || wire.ArrayT == nil {
+			return false
+		}
+		array := wire.ArrayT
+		return t.Len() == array.Len && dec.compatibleType(t.Elem(), array.Elem)
+	case *reflect.MapType:
+		wire, ok := dec.wireType[fw]
+		if !ok || wire.MapT == nil {
+			return false
+		}
+		MapType := wire.MapT
+		return dec.compatibleType(t.Key(), MapType.Key) && dec.compatibleType(t.Elem(), MapType.Elem)
+	case *reflect.SliceType:
+		// Is it an array of bytes?
+		if t.Elem().Kind() == reflect.Uint8 {
+			return fw == tBytes
+		}
+		// Extract and compare element types.
+		var sw *sliceType
+		if tt, ok := builtinIdToType[fw]; ok {
+			sw = tt.(*sliceType)
+		} else {
+			sw = dec.wireType[fw].SliceT
+		}
+		elem, _ := indirect(t.Elem())
+		return sw != nil && dec.compatibleType(elem, sw.Elem)
+	case *reflect.StructType:
+		return true
+	}
+	return true
+}
+
+// typeString returns a human-readable description of the type identified by remoteId.
+func (dec *Decoder) typeString(remoteId typeId) string {
+	if t := idToType[remoteId]; t != nil {
+		// globally known type.
+		return t.string()
+	}
+	return dec.wireType[remoteId].string()
+}
+
+
+func (dec *Decoder) compileSingle(remoteId typeId, rt reflect.Type) (engine *decEngine, err os.Error) {
+	engine = new(decEngine)
+	engine.instr = make([]decInstr, 1) // one item
+	name := rt.String()                // best we can do
+	if !dec.compatibleType(rt, remoteId) {
+		return nil, os.ErrorString("gob: wrong type received for local value " + name + ": " + dec.typeString(remoteId))
+	}
+	op, indir := dec.decOpFor(remoteId, rt, name)
+	ovfl := os.ErrorString(`value for "` + name + `" out of range`)
+	engine.instr[singletonField] = decInstr{op, singletonField, indir, 0, ovfl}
+	engine.numInstr = 1
+	return
+}
+
+// Is this an exported - upper case - name?
+func isExported(name string) bool {
+	rune, _ := utf8.DecodeRuneInString(name)
+	return unicode.IsUpper(rune)
+}
+
+func (dec *Decoder) compileDec(remoteId typeId, rt reflect.Type) (engine *decEngine, err os.Error) {
+	defer catchError(&err)
+	srt, ok := rt.(*reflect.StructType)
+	if !ok {
+		return dec.compileSingle(remoteId, rt)
+	}
+	var wireStruct *structType
+	// Builtin types can come from global pool; the rest must be defined by the decoder.
+	// Also we know we're decoding a struct now, so the client must have sent one.
+	if t, ok := builtinIdToType[remoteId]; ok {
+		wireStruct, _ = t.(*structType)
+	} else {
+		wireStruct = dec.wireType[remoteId].StructT
+	}
+	if wireStruct == nil {
+		errorf("gob: type mismatch in decoder: want struct type %s; got non-struct", rt.String())
+	}
+	engine = new(decEngine)
+	engine.instr = make([]decInstr, len(wireStruct.Field))
+	// Loop over the fields of the wire type.
+	for fieldnum := 0; fieldnum < len(wireStruct.Field); fieldnum++ {
+		wireField := wireStruct.Field[fieldnum]
+		if wireField.Name == "" {
+			errorf("gob: empty name for remote field of type %s", wireStruct.Name)
+		}
+		ovfl := overflow(wireField.Name)
+		// Find the field of the local type with the same name.
+		localField, present := srt.FieldByName(wireField.Name)
+		// TODO(r): anonymous names
+		if !present || !isExported(wireField.Name) {
+			op := dec.decIgnoreOpFor(wireField.Id)
+			engine.instr[fieldnum] = decInstr{op, fieldnum, 0, 0, ovfl}
+			continue
+		}
+		if !dec.compatibleType(localField.Type, wireField.Id) {
+			errorf("gob: wrong type (%s) for received field %s.%s", localField.Type, wireStruct.Name, wireField.Name)
+		}
+		op, indir := dec.decOpFor(wireField.Id, localField.Type, localField.Name)
+		engine.instr[fieldnum] = decInstr{op, fieldnum, indir, uintptr(localField.Offset), ovfl}
+		engine.numInstr++
+	}
+	return
+}
+
+func (dec *Decoder) getDecEnginePtr(remoteId typeId, rt reflect.Type) (enginePtr **decEngine, err os.Error) {
+	decoderMap, ok := dec.decoderCache[rt]
+	if !ok {
+		decoderMap = make(map[typeId]**decEngine)
+		dec.decoderCache[rt] = decoderMap
+	}
+	if enginePtr, ok = decoderMap[remoteId]; !ok {
+		// To handle recursive types, mark this engine as underway before compiling.
+		enginePtr = new(*decEngine)
+		decoderMap[remoteId] = enginePtr
+		*enginePtr, err = dec.compileDec(remoteId, rt)
+		if err != nil {
+			decoderMap[remoteId] = nil, false
+		}
+	}
+	return
+}
+
+// When ignoring struct data, in effect we compile it into this type
+type emptyStruct struct{}
+
+var emptyStructType = reflect.Typeof(emptyStruct{})
+
+func (dec *Decoder) getIgnoreEnginePtr(wireId typeId) (enginePtr **decEngine, err os.Error) {
+	var ok bool
+	if enginePtr, ok = dec.ignorerCache[wireId]; !ok {
+		// To handle recursive types, mark this engine as underway before compiling.
+		enginePtr = new(*decEngine)
+		dec.ignorerCache[wireId] = enginePtr
+		*enginePtr, err = dec.compileDec(wireId, emptyStructType)
+		if err != nil {
+			dec.ignorerCache[wireId] = nil, false
+		}
+	}
+	return
+}
+
+func (dec *Decoder) decode(wireId typeId, val reflect.Value) os.Error {
+	// Dereference down to the underlying struct type.
+	rt, indir := indirect(val.Type())
+	enginePtr, err := dec.getDecEnginePtr(wireId, rt)
+	if err != nil {
+		return err
+	}
+	engine := *enginePtr
+	if st, ok := rt.(*reflect.StructType); ok {
+		if engine.numInstr == 0 && st.NumField() > 0 && len(dec.wireType[wireId].StructT.Field) > 0 {
+			name := rt.Name()
+			return os.ErrorString("gob: type mismatch: no fields matched compiling decoder for " + name)
+		}
+		return dec.decodeStruct(engine, st, dec.state.b, uintptr(val.Addr()), indir)
+	}
+	return dec.decodeSingle(engine, rt, dec.state.b, uintptr(val.Addr()), indir)
+}
+
+func init() {
+	var iop, uop decOp
+	switch reflect.Typeof(int(0)).Bits() {
+	case 32:
+		iop = decInt32
+		uop = decUint32
+	case 64:
+		iop = decInt64
+		uop = decUint64
+	default:
+		panic("gob: unknown size of int/uint")
+	}
+	decOpMap[reflect.Int] = iop
+	decOpMap[reflect.Uint] = uop
+
+	// Finally uintptr
+	switch reflect.Typeof(uintptr(0)).Bits() {
+	case 32:
+		uop = decUint32
+	case 64:
+		uop = decUint64
+	default:
+		panic("gob: unknown size of uintptr")
+	}
+	decOpMap[reflect.Uintptr] = uop
+}
diff --git a/libgo/go/gob/decoder.go b/libgo/go/gob/decoder.go
new file mode 100644
index 000000000..664001a4b
--- /dev/null
+++ b/libgo/go/gob/decoder.go
@@ -0,0 +1,164 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package gob
+
+import (
+	"bytes"
+	"io"
+	"os"
+	"reflect"
+	"sync"
+)
+
+// A Decoder manages the receipt of type and data information read from the
+// remote side of a connection.
+type Decoder struct {
+	mutex        sync.Mutex                              // each item must be received atomically
+	r            io.Reader                               // source of the data
+	wireType     map[typeId]*wireType                    // map from remote ID to local description
+	decoderCache map[reflect.Type]map[typeId]**decEngine // cache of compiled engines
+	ignorerCache map[typeId]**decEngine                  // ditto for ignored objects
+	state        *decodeState                            // reads data from in-memory buffer
+	countState   *decodeState                            // reads counts from wire
+	buf          []byte
+	countBuf     [9]byte // counts may be uint64s (unlikely!), require 9 bytes
+	byteBuffer   *bytes.Buffer
+	err          os.Error
+}
+
+// NewDecoder returns a new decoder that reads from the io.Reader.
+func NewDecoder(r io.Reader) *Decoder {
+	dec := new(Decoder)
+	dec.r = r
+	dec.wireType = make(map[typeId]*wireType)
+	dec.state = newDecodeState(dec, &dec.byteBuffer) // buffer set in Decode()
+	dec.decoderCache = make(map[reflect.Type]map[typeId]**decEngine)
+	dec.ignorerCache = make(map[typeId]**decEngine)
+
+	return dec
+}
+
+// recvType loads the definition of a type and reloads the Decoder's buffer.
+func (dec *Decoder) recvType(id typeId) {
+	// Have we already seen this type?  That's an error
+	if dec.wireType[id] != nil {
+		dec.err = os.ErrorString("gob: duplicate type received")
+		return
+	}
+
+	// Type:
+	wire := new(wireType)
+	dec.err = dec.decode(tWireType, reflect.NewValue(wire))
+	if dec.err != nil {
+		return
+	}
+	// Remember we've seen this type.
+	dec.wireType[id] = wire
+
+	// Load the next parcel.
+	dec.recv()
+}
+
+// Decode reads the next value from the connection and stores
+// it in the data represented by the empty interface value.
+// The value underlying e must be the correct type for the next
+// data item received, and must be a pointer.
+func (dec *Decoder) Decode(e interface{}) os.Error {
+	value := reflect.NewValue(e)
+	// If e represents a value as opposed to a pointer, the answer won't
+	// get back to the caller.  Make sure it's a pointer.
+	if value.Type().Kind() != reflect.Ptr {
+		dec.err = os.ErrorString("gob: attempt to decode into a non-pointer")
+		return dec.err
+	}
+	return dec.DecodeValue(value)
+}
+
+// recv reads the next count-delimited item from the input. It is the converse
+// of Encoder.send.
+func (dec *Decoder) recv() {
+	// Read a count.
+	var nbytes uint64
+	nbytes, dec.err = decodeUintReader(dec.r, dec.countBuf[0:])
+	if dec.err != nil {
+		return
+	}
+	// Allocate the buffer.
+	if nbytes > uint64(len(dec.buf)) {
+		dec.buf = make([]byte, nbytes+1000)
+	}
+	dec.byteBuffer = bytes.NewBuffer(dec.buf[0:nbytes])
+
+	// Read the data
+	_, dec.err = io.ReadFull(dec.r, dec.buf[0:nbytes])
+	if dec.err != nil {
+		if dec.err == os.EOF {
+			dec.err = io.ErrUnexpectedEOF
+		}
+		return
+	}
+}
+
+// decodeValueFromBuffer grabs the next value from the input. The Decoder's
+// buffer already contains data.  If the next item in the buffer is a type
+// descriptor, it may be necessary to reload the buffer, but recvType does that.
+func (dec *Decoder) decodeValueFromBuffer(value reflect.Value, ignoreInterfaceValue, countPresent bool) {
+	for dec.state.b.Len() > 0 {
+		// Receive a type id.
+		id := typeId(dec.state.decodeInt())
+
+		// Is it a new type?
+		if id < 0 { // 0 is the error state, handled above
+			// If the id is negative, we have a type.
+			dec.recvType(-id)
+			if dec.err != nil {
+				break
+			}
+			continue
+		}
+
+		// Make sure the type has been defined already or is a builtin type (for
+		// top-level singleton values).
+		if dec.wireType[id] == nil && builtinIdToType[id] == nil {
+			dec.err = errBadType
+			break
+		}
+		// An interface value is preceded by a byte count.
+		if countPresent {
+			count := int(dec.state.decodeUint())
+			if ignoreInterfaceValue {
+				// An interface value is preceded by a byte count. Just skip that many bytes.
+				dec.state.b.Next(int(count))
+				break
+			}
+			// Otherwise fall through and decode it.
+		}
+		dec.err = dec.decode(id, value)
+		break
+	}
+}
+
+// DecodeValue reads the next value from the connection and stores
+// it in the data represented by the reflection value.
+// The value must be the correct type for the next
+// data item received.
+func (dec *Decoder) DecodeValue(value reflect.Value) os.Error {
+	// Make sure we're single-threaded through here.
+	dec.mutex.Lock()
+	defer dec.mutex.Unlock()
+
+	dec.err = nil
+	dec.recv()
+	if dec.err != nil {
+		return dec.err
+	}
+	dec.decodeValueFromBuffer(value, false, false)
+	return dec.err
+}
+
+// If debug.go is compiled into the program , debugFunc prints a human-readable
+// representation of the gob data read from r by calling that file's Debug function.
+// Otherwise it is nil.
+var debugFunc func(io.Reader)
diff --git a/libgo/go/gob/doc.go b/libgo/go/gob/doc.go
new file mode 100644
index 000000000..31253f16d
--- /dev/null
+++ b/libgo/go/gob/doc.go
@@ -0,0 +1,307 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+/*
+The gob package manages streams of gobs - binary values exchanged between an
+Encoder (transmitter) and a Decoder (receiver).  A typical use is transporting
+arguments and results of remote procedure calls (RPCs) such as those provided by
+package "rpc".
+
+A stream of gobs is self-describing.  Each data item in the stream is preceded by
+a specification of its type, expressed in terms of a small set of predefined
+types.  Pointers are not transmitted, but the things they point to are
+transmitted; that is, the values are flattened.  Recursive types work fine, but
+recursive values (data with cycles) are problematic.  This may change.
+
+To use gobs, create an Encoder and present it with a series of data items as
+values or addresses that can be dereferenced to values.  The Encoder makes sure
+all type information is sent before it is needed.  At the receive side, a
+Decoder retrieves values from the encoded stream and unpacks them into local
+variables.
+
+The source and destination values/types need not correspond exactly.  For structs,
+fields (identified by name) that are in the source but absent from the receiving
+variable will be ignored.  Fields that are in the receiving variable but missing
+from the transmitted type or value will be ignored in the destination.  If a field
+with the same name is present in both, their types must be compatible. Both the
+receiver and transmitter will do all necessary indirection and dereferencing to
+convert between gobs and actual Go values.  For instance, a gob type that is
+schematically,
+
+	struct { a, b int }
+
+can be sent from or received into any of these Go types:
+
+	struct { a, b int }	// the same
+	*struct { a, b int }	// extra indirection of the struct
+	struct { *a, **b int }	// extra indirection of the fields
+	struct { a, b int64 }	// different concrete value type; see below
+
+It may also be received into any of these:
+
+	struct { a, b int }	// the same
+	struct { b, a int }	// ordering doesn't matter; matching is by name
+	struct { a, b, c int }	// extra field (c) ignored
+	struct { b int }	// missing field (a) ignored; data will be dropped
+	struct { b, c int }	// missing field (a) ignored; extra field (c) ignored.
+
+Attempting to receive into these types will draw a decode error:
+
+	struct { a int; b uint }	// change of signedness for b
+	struct { a int; b float }	// change of type for b
+	struct { }			// no field names in common
+	struct { c, d int }		// no field names in common
+
+Integers are transmitted two ways: arbitrary precision signed integers or
+arbitrary precision unsigned integers.  There is no int8, int16 etc.
+discrimination in the gob format; there are only signed and unsigned integers.  As
+described below, the transmitter sends the value in a variable-length encoding;
+the receiver accepts the value and stores it in the destination variable.
+Floating-point numbers are always sent using IEEE-754 64-bit precision (see
+below).
+
+Signed integers may be received into any signed integer variable: int, int16, etc.;
+unsigned integers may be received into any unsigned integer variable; and floating
+point values may be received into any floating point variable.  However,
+the destination variable must be able to represent the value or the decode
+operation will fail.
+
+Structs, arrays and slices are also supported.  Strings and arrays of bytes are
+supported with a special, efficient representation (see below).
+
+Functions and channels cannot be sent in a gob.  Attempting
+to encode a value that contains one will fail.
+
+The rest of this comment documents the encoding, details that are not important
+for most users.  Details are presented bottom-up.
+
+An unsigned integer is sent one of two ways.  If it is less than 128, it is sent
+as a byte with that value.  Otherwise it is sent as a minimal-length big-endian
+(high byte first) byte stream holding the value, preceded by one byte holding the
+byte count, negated.  Thus 0 is transmitted as (00), 7 is transmitted as (07) and
+256 is transmitted as (FE 01 00).
+
+A boolean is encoded within an unsigned integer: 0 for false, 1 for true.
+
+A signed integer, i, is encoded within an unsigned integer, u.  Within u, bits 1
+upward contain the value; bit 0 says whether they should be complemented upon
+receipt.  The encode algorithm looks like this:
+
+	uint u;
+	if i < 0 {
+		u = (^i << 1) | 1	// complement i, bit 0 is 1
+	} else {
+		u = (i << 1)	// do not complement i, bit 0 is 0
+	}
+	encodeUnsigned(u)
+
+The low bit is therefore analogous to a sign bit, but making it the complement bit
+instead guarantees that the largest negative integer is not a special case.  For
+example, -129=^128=(^256>>1) encodes as (FE 01 01).
+
+Floating-point numbers are always sent as a representation of a float64 value.
+That value is converted to a uint64 using math.Float64bits.  The uint64 is then
+byte-reversed and sent as a regular unsigned integer.  The byte-reversal means the
+exponent and high-precision part of the mantissa go first.  Since the low bits are
+often zero, this can save encoding bytes.  For instance, 17.0 is encoded in only
+three bytes (FE 31 40).
+
+Strings and slices of bytes are sent as an unsigned count followed by that many
+uninterpreted bytes of the value.
+
+All other slices and arrays are sent as an unsigned count followed by that many
+elements using the standard gob encoding for their type, recursively.
+
+Structs are sent as a sequence of (field number, field value) pairs.  The field
+value is sent using the standard gob encoding for its type, recursively.  If a
+field has the zero value for its type, it is omitted from the transmission.  The
+field number is defined by the type of the encoded struct: the first field of the
+encoded type is field 0, the second is field 1, etc.  When encoding a value, the
+field numbers are delta encoded for efficiency and the fields are always sent in
+order of increasing field number; the deltas are therefore unsigned.  The
+initialization for the delta encoding sets the field number to -1, so an unsigned
+integer field 0 with value 7 is transmitted as unsigned delta = 1, unsigned value
+= 7 or (01 07).  Finally, after all the fields have been sent a terminating mark
+denotes the end of the struct.  That mark is a delta=0 value, which has
+representation (00).
+
+Interface types are not checked for compatibility; all interface types are
+treated, for transmission, as members of a single "interface" type, analogous to
+int or []byte - in effect they're all treated as interface{}.  Interface values
+are transmitted as a string identifying the concrete type being sent (a name
+that must be pre-defined by calling Register), followed by a byte count of the
+length of the following data (so the value can be skipped if it cannot be
+stored), followed by the usual encoding of concrete (dynamic) value stored in
+the interface value.  (A nil interface value is identified by the empty string
+and transmits no value.) Upon receipt, the decoder verifies that the unpacked
+concrete item satisfies the interface of the receiving variable.
+
+The representation of types is described below.  When a type is defined on a given
+connection between an Encoder and Decoder, it is assigned a signed integer type
+id.  When Encoder.Encode(v) is called, it makes sure there is an id assigned for
+the type of v and all its elements and then it sends the pair (typeid, encoded-v)
+where typeid is the type id of the encoded type of v and encoded-v is the gob
+encoding of the value v.
+
+To define a type, the encoder chooses an unused, positive type id and sends the
+pair (-type id, encoded-type) where encoded-type is the gob encoding of a wireType
+description, constructed from these types:
+
+	type wireType struct {
+		ArrayT  *ArrayType
+		SliceT  *SliceType
+		StructT *StructType
+		MapT    *MapType
+	}
+	type ArrayType struct {
+		CommonType
+		Elem typeId
+		Len  int
+	}
+	type CommonType {
+		Name string // the name of the struct type
+		Id  int    // the id of the type, repeated so it's inside the type
+	}
+	type SliceType struct {
+		CommonType
+		Elem typeId
+	}
+	type StructType struct {
+		CommonType
+		Field []*fieldType // the fields of the struct.
+	}
+	type FieldType struct {
+		Name string // the name of the field.
+		Id   int    // the type id of the field, which must be already defined
+	}
+	type MapType struct {
+		CommonType
+		Key  typeId
+		Elem typeId
+	}
+
+If there are nested type ids, the types for all inner type ids must be defined
+before the top-level type id is used to describe an encoded-v.
+
+For simplicity in setup, the connection is defined to understand these types a
+priori, as well as the basic gob types int, uint, etc.  Their ids are:
+
+	bool        1
+	int         2
+	uint        3
+	float       4
+	[]byte      5
+	string      6
+	complex     7
+	interface   8
+	// gap for reserved ids.
+	WireType    16
+	ArrayType   17
+	CommonType  18
+	SliceType   19
+	StructType  20
+	FieldType   21
+	// 22 is slice of fieldType.
+	MapType     23
+
+Finally, each message created by a call to Encode is preceded by an encoded
+unsigned integer count of the number of bytes remaining in the message.  After
+the initial type name, interface values are wrapped the same way; in effect, the
+interface value acts like a recursive invocation of Encode.
+
+In summary, a gob stream looks like
+
+	(byteCount (-type id, encoding of a wireType)* (type id, encoding of a value))*
+
+where * signifies zero or more repetitions and the type id of a value must
+be predefined or be defined before the value in the stream.
+*/
+package gob
+
+/*
+For implementers and the curious, here is an encoded example.  Given
+	type Point struct {x, y int}
+and the value
+	p := Point{22, 33}
+the bytes transmitted that encode p will be:
+	1f ff 81 03 01 01 05 50 6f 69 6e 74 01 ff 82 00
+	01 02 01 01 78 01 04 00 01 01 79 01 04 00 00 00
+	07 ff 82 01 2c 01 42 00
+They are determined as follows.
+
+Since this is the first transmission of type Point, the type descriptor
+for Point itself must be sent before the value.  This is the first type
+we've sent on this Encoder, so it has type id 65 (0 through 64 are
+reserved).
+
+	1f	// This item (a type descriptor) is 31 bytes long.
+	ff 81	// The negative of the id for the type we're defining, -65.
+		// This is one byte (indicated by FF = -1) followed by
+		// ^-65<<1 | 1.  The low 1 bit signals to complement the
+		// rest upon receipt.
+
+	// Now we send a type descriptor, which is itself a struct (wireType).
+	// The type of wireType itself is known (it's built in, as is the type of
+	// all its components), so we just need to send a *value* of type wireType
+	// that represents type "Point".
+	// Here starts the encoding of that value.
+	// Set the field number implicitly to -1; this is done at the beginning
+	// of every struct, including nested structs.
+	03	// Add 3 to field number; now 2 (wireType.structType; this is a struct).
+		// structType starts with an embedded commonType, which appears
+		// as a regular structure here too.
+	01	// add 1 to field number (now 0); start of embedded commonType.
+	01	// add 1 to field number (now 0, the name of the type)
+	05	// string is (unsigned) 5 bytes long
+	50 6f 69 6e 74	// wireType.structType.commonType.name = "Point"
+	01	// add 1 to field number (now 1, the id of the type)
+	ff 82	// wireType.structType.commonType._id = 65
+	00	// end of embedded wiretype.structType.commonType struct
+	01	// add 1 to field number (now 1, the field array in wireType.structType)
+	02	// There are two fields in the type (len(structType.field))
+	01	// Start of first field structure; add 1 to get field number 0: field[0].name
+	01	// 1 byte
+	78	// structType.field[0].name = "x"
+	01	// Add 1 to get field number 1: field[0].id
+	04	// structType.field[0].typeId is 2 (signed int).
+	00	// End of structType.field[0]; start structType.field[1]; set field number to -1.
+	01	// Add 1 to get field number 0: field[1].name
+	01	// 1 byte
+	79	// structType.field[1].name = "y"
+	01	// Add 1 to get field number 1: field[0].id
+	04	// struct.Type.field[1].typeId is 2 (signed int).
+	00	// End of structType.field[1]; end of structType.field.
+	00	// end of wireType.structType structure
+	00	// end of wireType structure
+
+Now we can send the Point value.  Again the field number resets to -1:
+
+	07	// this value is 7 bytes long
+	ff 82	// the type number, 65 (1 byte (-FF) followed by 65<<1)
+	01	// add one to field number, yielding field 0
+	2c	// encoding of signed "22" (0x22 = 44 = 22<<1); Point.x = 22
+	01	// add one to field number, yielding field 1
+	42	// encoding of signed "33" (0x42 = 66 = 33<<1); Point.y = 33
+	00	// end of structure
+
+The type encoding is long and fairly intricate but we send it only once.
+If p is transmitted a second time, the type is already known so the
+output will be just:
+
+	07 ff 82 01 2c 01 42 00
+
+A single non-struct value at top level is transmitted like a field with
+delta tag 0.  For instance, a signed integer with value 3 presented as
+the argument to Encode will emit:
+
+	03 04 00 06
+
+Which represents:
+
+	03	// this value is 3 bytes long
+	04	// the type number, 2, represents an integer
+	00	// tag delta 0
+	06	// value 3
+
+*/
diff --git a/libgo/go/gob/encode.go b/libgo/go/gob/encode.go
new file mode 100644
index 000000000..d286a7e00
--- /dev/null
+++ b/libgo/go/gob/encode.go
@@ -0,0 +1,573 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package gob
+
+import (
+	"bytes"
+	"io"
+	"math"
+	"os"
+	"reflect"
+	"unsafe"
+)
+
+const uint64Size = unsafe.Sizeof(uint64(0))
+
+// The global execution state of an instance of the encoder.
+// Field numbers are delta encoded and always increase. The field
+// number is initialized to -1 so 0 comes out as delta(1). A delta of
+// 0 terminates the structure.
+type encoderState struct {
+	enc      *Encoder
+	b        *bytes.Buffer
+	sendZero bool                 // encoding an array element or map key/value pair; send zero values
+	fieldnum int                  // the last field number written.
+	buf      [1 + uint64Size]byte // buffer used by the encoder; here to avoid allocation.
+}
+
+func newEncoderState(enc *Encoder, b *bytes.Buffer) *encoderState {
+	return &encoderState{enc: enc, b: b}
+}
+
+// Unsigned integers have a two-state encoding.  If the number is less
+// than 128 (0 through 0x7F), its value is written directly.
+// Otherwise the value is written in big-endian byte order preceded
+// by the byte length, negated.
+
+// encodeUint writes an encoded unsigned integer to state.b.
+func (state *encoderState) encodeUint(x uint64) {
+	if x <= 0x7F {
+		err := state.b.WriteByte(uint8(x))
+		if err != nil {
+			error(err)
+		}
+		return
+	}
+	var n, m int
+	m = uint64Size
+	for n = 1; x > 0; n++ {
+		state.buf[m] = uint8(x & 0xFF)
+		x >>= 8
+		m--
+	}
+	state.buf[m] = uint8(-(n - 1))
+	n, err := state.b.Write(state.buf[m : uint64Size+1])
+	if err != nil {
+		error(err)
+	}
+}
+
+// encodeInt writes an encoded signed integer to state.w.
+// The low bit of the encoding says whether to bit complement the (other bits of the)
+// uint to recover the int.
+func (state *encoderState) encodeInt(i int64) {
+	var x uint64
+	if i < 0 {
+		x = uint64(^i<<1) | 1
+	} else {
+		x = uint64(i << 1)
+	}
+	state.encodeUint(uint64(x))
+}
+
+type encOp func(i *encInstr, state *encoderState, p unsafe.Pointer)
+
+// The 'instructions' of the encoding machine
+type encInstr struct {
+	op     encOp
+	field  int     // field number
+	indir  int     // how many pointer indirections to reach the value in the struct
+	offset uintptr // offset in the structure of the field to encode
+}
+
+// Emit a field number and update the state to record its value for delta encoding.
+// If the instruction pointer is nil, do nothing
+func (state *encoderState) update(instr *encInstr) {
+	if instr != nil {
+		state.encodeUint(uint64(instr.field - state.fieldnum))
+		state.fieldnum = instr.field
+	}
+}
+
+// Each encoder is responsible for handling any indirections associated
+// with the data structure.  If any pointer so reached is nil, no bytes are written.
+// If the data item is zero, no bytes are written.
+// Otherwise, the output (for a scalar) is the field number, as an encoded integer,
+// followed by the field data in its appropriate format.
+
+func encIndirect(p unsafe.Pointer, indir int) unsafe.Pointer {
+	for ; indir > 0; indir-- {
+		p = *(*unsafe.Pointer)(p)
+		if p == nil {
+			return unsafe.Pointer(nil)
+		}
+	}
+	return p
+}
+
+func encBool(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	b := *(*bool)(p)
+	if b || state.sendZero {
+		state.update(i)
+		if b {
+			state.encodeUint(1)
+		} else {
+			state.encodeUint(0)
+		}
+	}
+}
+
+func encInt(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	v := int64(*(*int)(p))
+	if v != 0 || state.sendZero {
+		state.update(i)
+		state.encodeInt(v)
+	}
+}
+
+func encUint(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	v := uint64(*(*uint)(p))
+	if v != 0 || state.sendZero {
+		state.update(i)
+		state.encodeUint(v)
+	}
+}
+
+func encInt8(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	v := int64(*(*int8)(p))
+	if v != 0 || state.sendZero {
+		state.update(i)
+		state.encodeInt(v)
+	}
+}
+
+func encUint8(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	v := uint64(*(*uint8)(p))
+	if v != 0 || state.sendZero {
+		state.update(i)
+		state.encodeUint(v)
+	}
+}
+
+func encInt16(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	v := int64(*(*int16)(p))
+	if v != 0 || state.sendZero {
+		state.update(i)
+		state.encodeInt(v)
+	}
+}
+
+func encUint16(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	v := uint64(*(*uint16)(p))
+	if v != 0 || state.sendZero {
+		state.update(i)
+		state.encodeUint(v)
+	}
+}
+
+func encInt32(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	v := int64(*(*int32)(p))
+	if v != 0 || state.sendZero {
+		state.update(i)
+		state.encodeInt(v)
+	}
+}
+
+func encUint32(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	v := uint64(*(*uint32)(p))
+	if v != 0 || state.sendZero {
+		state.update(i)
+		state.encodeUint(v)
+	}
+}
+
+func encInt64(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	v := *(*int64)(p)
+	if v != 0 || state.sendZero {
+		state.update(i)
+		state.encodeInt(v)
+	}
+}
+
+func encUint64(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	v := *(*uint64)(p)
+	if v != 0 || state.sendZero {
+		state.update(i)
+		state.encodeUint(v)
+	}
+}
+
+func encUintptr(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	v := uint64(*(*uintptr)(p))
+	if v != 0 || state.sendZero {
+		state.update(i)
+		state.encodeUint(v)
+	}
+}
+
+// Floating-point numbers are transmitted as uint64s holding the bits
+// of the underlying representation.  They are sent byte-reversed, with
+// the exponent end coming out first, so integer floating point numbers
+// (for example) transmit more compactly.  This routine does the
+// swizzling.
+func floatBits(f float64) uint64 {
+	u := math.Float64bits(f)
+	var v uint64
+	for i := 0; i < 8; i++ {
+		v <<= 8
+		v |= u & 0xFF
+		u >>= 8
+	}
+	return v
+}
+
+func encFloat32(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	f := *(*float32)(p)
+	if f != 0 || state.sendZero {
+		v := floatBits(float64(f))
+		state.update(i)
+		state.encodeUint(v)
+	}
+}
+
+func encFloat64(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	f := *(*float64)(p)
+	if f != 0 || state.sendZero {
+		state.update(i)
+		v := floatBits(f)
+		state.encodeUint(v)
+	}
+}
+
+// Complex numbers are just a pair of floating-point numbers, real part first.
+func encComplex64(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	c := *(*complex64)(p)
+	if c != 0+0i || state.sendZero {
+		rpart := floatBits(float64(real(c)))
+		ipart := floatBits(float64(imag(c)))
+		state.update(i)
+		state.encodeUint(rpart)
+		state.encodeUint(ipart)
+	}
+}
+
+func encComplex128(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	c := *(*complex128)(p)
+	if c != 0+0i || state.sendZero {
+		rpart := floatBits(real(c))
+		ipart := floatBits(imag(c))
+		state.update(i)
+		state.encodeUint(rpart)
+		state.encodeUint(ipart)
+	}
+}
+
+func encNoOp(i *encInstr, state *encoderState, p unsafe.Pointer) {
+}
+
+// Byte arrays are encoded as an unsigned count followed by the raw bytes.
+func encUint8Array(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	b := *(*[]byte)(p)
+	if len(b) > 0 || state.sendZero {
+		state.update(i)
+		state.encodeUint(uint64(len(b)))
+		state.b.Write(b)
+	}
+}
+
+// Strings are encoded as an unsigned count followed by the raw bytes.
+func encString(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	s := *(*string)(p)
+	if len(s) > 0 || state.sendZero {
+		state.update(i)
+		state.encodeUint(uint64(len(s)))
+		io.WriteString(state.b, s)
+	}
+}
+
+// The end of a struct is marked by a delta field number of 0.
+func encStructTerminator(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	state.encodeUint(0)
+}
+
+// Execution engine
+
+// The encoder engine is an array of instructions indexed by field number of the encoding
+// data, typically a struct.  It is executed top to bottom, walking the struct.
+type encEngine struct {
+	instr []encInstr
+}
+
+const singletonField = 0
+
+func (enc *Encoder) encodeSingle(b *bytes.Buffer, engine *encEngine, basep uintptr) {
+	state := newEncoderState(enc, b)
+	state.fieldnum = singletonField
+	// There is no surrounding struct to frame the transmission, so we must
+	// generate data even if the item is zero.  To do this, set sendZero.
+	state.sendZero = true
+	instr := &engine.instr[singletonField]
+	p := unsafe.Pointer(basep) // offset will be zero
+	if instr.indir > 0 {
+		if p = encIndirect(p, instr.indir); p == nil {
+			return
+		}
+	}
+	instr.op(instr, state, p)
+}
+
+func (enc *Encoder) encodeStruct(b *bytes.Buffer, engine *encEngine, basep uintptr) {
+	state := newEncoderState(enc, b)
+	state.fieldnum = -1
+	for i := 0; i < len(engine.instr); i++ {
+		instr := &engine.instr[i]
+		p := unsafe.Pointer(basep + instr.offset)
+		if instr.indir > 0 {
+			if p = encIndirect(p, instr.indir); p == nil {
+				continue
+			}
+		}
+		instr.op(instr, state, p)
+	}
+}
+
+func (enc *Encoder) encodeArray(b *bytes.Buffer, p uintptr, op encOp, elemWid uintptr, elemIndir int, length int) {
+	state := newEncoderState(enc, b)
+	state.fieldnum = -1
+	state.sendZero = true
+	state.encodeUint(uint64(length))
+	for i := 0; i < length; i++ {
+		elemp := p
+		up := unsafe.Pointer(elemp)
+		if elemIndir > 0 {
+			if up = encIndirect(up, elemIndir); up == nil {
+				errorf("gob: encodeArray: nil element")
+			}
+			elemp = uintptr(up)
+		}
+		op(nil, state, unsafe.Pointer(elemp))
+		p += uintptr(elemWid)
+	}
+}
+
+func encodeReflectValue(state *encoderState, v reflect.Value, op encOp, indir int) {
+	for i := 0; i < indir && v != nil; i++ {
+		v = reflect.Indirect(v)
+	}
+	if v == nil {
+		errorf("gob: encodeReflectValue: nil element")
+	}
+	op(nil, state, unsafe.Pointer(v.Addr()))
+}
+
+func (enc *Encoder) encodeMap(b *bytes.Buffer, mv *reflect.MapValue, keyOp, elemOp encOp, keyIndir, elemIndir int) {
+	state := newEncoderState(enc, b)
+	state.fieldnum = -1
+	state.sendZero = true
+	keys := mv.Keys()
+	state.encodeUint(uint64(len(keys)))
+	for _, key := range keys {
+		encodeReflectValue(state, key, keyOp, keyIndir)
+		encodeReflectValue(state, mv.Elem(key), elemOp, elemIndir)
+	}
+}
+
+// To send an interface, we send a string identifying the concrete type, followed
+// by the type identifier (which might require defining that type right now), followed
+// by the concrete value.  A nil value gets sent as the empty string for the name,
+// followed by no value.
+func (enc *Encoder) encodeInterface(b *bytes.Buffer, iv *reflect.InterfaceValue) {
+	state := newEncoderState(enc, b)
+	state.fieldnum = -1
+	state.sendZero = true
+	if iv.IsNil() {
+		state.encodeUint(0)
+		return
+	}
+
+	typ, _ := indirect(iv.Elem().Type())
+	name, ok := concreteTypeToName[typ]
+	if !ok {
+		errorf("gob: type not registered for interface: %s", typ)
+	}
+	// Send the name.
+	state.encodeUint(uint64(len(name)))
+	_, err := io.WriteString(state.b, name)
+	if err != nil {
+		error(err)
+	}
+	// Send (and maybe first define) the type id.
+	enc.sendTypeDescriptor(typ)
+	// Encode the value into a new buffer.
+	data := new(bytes.Buffer)
+	err = enc.encode(data, iv.Elem())
+	if err != nil {
+		error(err)
+	}
+	state.encodeUint(uint64(data.Len()))
+	_, err = state.b.Write(data.Bytes())
+	if err != nil {
+		error(err)
+	}
+}
+
+var encOpMap = []encOp{
+	reflect.Bool:       encBool,
+	reflect.Int:        encInt,
+	reflect.Int8:       encInt8,
+	reflect.Int16:      encInt16,
+	reflect.Int32:      encInt32,
+	reflect.Int64:      encInt64,
+	reflect.Uint:       encUint,
+	reflect.Uint8:      encUint8,
+	reflect.Uint16:     encUint16,
+	reflect.Uint32:     encUint32,
+	reflect.Uint64:     encUint64,
+	reflect.Uintptr:    encUintptr,
+	reflect.Float32:    encFloat32,
+	reflect.Float64:    encFloat64,
+	reflect.Complex64:  encComplex64,
+	reflect.Complex128: encComplex128,
+	reflect.String:     encString,
+}
+
+// Return the encoding op for the base type under rt and
+// the indirection count to reach it.
+func (enc *Encoder) encOpFor(rt reflect.Type) (encOp, int) {
+	typ, indir := indirect(rt)
+	var op encOp
+	k := typ.Kind()
+	if int(k) < len(encOpMap) {
+		op = encOpMap[k]
+	}
+	if op == nil {
+		// Special cases
+		switch t := typ.(type) {
+		case *reflect.SliceType:
+			if t.Elem().Kind() == reflect.Uint8 {
+				op = encUint8Array
+				break
+			}
+			// Slices have a header; we decode it to find the underlying array.
+			elemOp, indir := enc.encOpFor(t.Elem())
+			op = func(i *encInstr, state *encoderState, p unsafe.Pointer) {
+				slice := (*reflect.SliceHeader)(p)
+				if !state.sendZero && slice.Len == 0 {
+					return
+				}
+				state.update(i)
+				state.enc.encodeArray(state.b, slice.Data, elemOp, t.Elem().Size(), indir, int(slice.Len))
+			}
+		case *reflect.ArrayType:
+			// True arrays have size in the type.
+			elemOp, indir := enc.encOpFor(t.Elem())
+			op = func(i *encInstr, state *encoderState, p unsafe.Pointer) {
+				state.update(i)
+				state.enc.encodeArray(state.b, uintptr(p), elemOp, t.Elem().Size(), indir, t.Len())
+			}
+		case *reflect.MapType:
+			keyOp, keyIndir := enc.encOpFor(t.Key())
+			elemOp, elemIndir := enc.encOpFor(t.Elem())
+			op = func(i *encInstr, state *encoderState, p unsafe.Pointer) {
+				// Maps cannot be accessed by moving addresses around the way
+				// that slices etc. can.  We must recover a full reflection value for
+				// the iteration.
+				v := reflect.NewValue(unsafe.Unreflect(t, unsafe.Pointer((p))))
+				mv := reflect.Indirect(v).(*reflect.MapValue)
+				if !state.sendZero && mv.Len() == 0 {
+					return
+				}
+				state.update(i)
+				state.enc.encodeMap(state.b, mv, keyOp, elemOp, keyIndir, elemIndir)
+			}
+		case *reflect.StructType:
+			// Generate a closure that calls out to the engine for the nested type.
+			enc.getEncEngine(typ)
+			info := mustGetTypeInfo(typ)
+			op = func(i *encInstr, state *encoderState, p unsafe.Pointer) {
+				state.update(i)
+				// indirect through info to delay evaluation for recursive structs
+				state.enc.encodeStruct(state.b, info.encoder, uintptr(p))
+			}
+		case *reflect.InterfaceType:
+			op = func(i *encInstr, state *encoderState, p unsafe.Pointer) {
+				// Interfaces transmit the name and contents of the concrete
+				// value they contain.
+				v := reflect.NewValue(unsafe.Unreflect(t, unsafe.Pointer((p))))
+				iv := reflect.Indirect(v).(*reflect.InterfaceValue)
+				if !state.sendZero && (iv == nil || iv.IsNil()) {
+					return
+				}
+				state.update(i)
+				state.enc.encodeInterface(state.b, iv)
+			}
+		}
+	}
+	if op == nil {
+		errorf("gob enc: can't happen: encode type %s", rt.String())
+	}
+	return op, indir
+}
+
+// The local Type was compiled from the actual value, so we know it's compatible.
+func (enc *Encoder) compileEnc(rt reflect.Type) *encEngine {
+	srt, isStruct := rt.(*reflect.StructType)
+	engine := new(encEngine)
+	if isStruct {
+		engine.instr = make([]encInstr, srt.NumField()+1) // +1 for terminator
+		for fieldnum := 0; fieldnum < srt.NumField(); fieldnum++ {
+			f := srt.Field(fieldnum)
+			op, indir := enc.encOpFor(f.Type)
+			if !isExported(f.Name) {
+				op = encNoOp
+			}
+			engine.instr[fieldnum] = encInstr{op, fieldnum, indir, uintptr(f.Offset)}
+		}
+		engine.instr[srt.NumField()] = encInstr{encStructTerminator, 0, 0, 0}
+	} else {
+		engine.instr = make([]encInstr, 1)
+		op, indir := enc.encOpFor(rt)
+		engine.instr[0] = encInstr{op, singletonField, indir, 0} // offset is zero
+	}
+	return engine
+}
+
+// typeLock must be held (or we're in initialization and guaranteed single-threaded).
+// The reflection type must have all its indirections processed out.
+func (enc *Encoder) getEncEngine(rt reflect.Type) *encEngine {
+	info, err1 := getTypeInfo(rt)
+	if err1 != nil {
+		error(err1)
+	}
+	if info.encoder == nil {
+		// mark this engine as underway before compiling to handle recursive types.
+		info.encoder = new(encEngine)
+		info.encoder = enc.compileEnc(rt)
+	}
+	return info.encoder
+}
+
+// Put this in a function so we can hold the lock only while compiling, not when encoding.
+func (enc *Encoder) lockAndGetEncEngine(rt reflect.Type) *encEngine {
+	typeLock.Lock()
+	defer typeLock.Unlock()
+	return enc.getEncEngine(rt)
+}
+
+func (enc *Encoder) encode(b *bytes.Buffer, value reflect.Value) (err os.Error) {
+	defer catchError(&err)
+	// Dereference down to the underlying object.
+	rt, indir := indirect(value.Type())
+	for i := 0; i < indir; i++ {
+		value = reflect.Indirect(value)
+	}
+	engine := enc.lockAndGetEncEngine(rt)
+	if value.Type().Kind() == reflect.Struct {
+		enc.encodeStruct(b, engine, value.Addr())
+	} else {
+		enc.encodeSingle(b, engine, value.Addr())
+	}
+	return nil
+}
diff --git a/libgo/go/gob/encoder.go b/libgo/go/gob/encoder.go
new file mode 100644
index 000000000..8869b2629
--- /dev/null
+++ b/libgo/go/gob/encoder.go
@@ -0,0 +1,207 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package gob
+
+import (
+	"bytes"
+	"io"
+	"os"
+	"reflect"
+	"sync"
+)
+
+// An Encoder manages the transmission of type and data information to the
+// other side of a connection.
+type Encoder struct {
+	mutex      sync.Mutex              // each item must be sent atomically
+	w          io.Writer               // where to send the data
+	sent       map[reflect.Type]typeId // which types we've already sent
+	state      *encoderState           // so we can encode integers, strings directly
+	countState *encoderState           // stage for writing counts
+	buf        []byte                  // for collecting the output.
+	err        os.Error
+}
+
+// NewEncoder returns a new encoder that will transmit on the io.Writer.
+func NewEncoder(w io.Writer) *Encoder {
+	enc := new(Encoder)
+	enc.w = w
+	enc.sent = make(map[reflect.Type]typeId)
+	enc.state = newEncoderState(enc, new(bytes.Buffer))
+	enc.countState = newEncoderState(enc, new(bytes.Buffer))
+	return enc
+}
+
+func (enc *Encoder) badType(rt reflect.Type) {
+	enc.setError(os.ErrorString("gob: can't encode type " + rt.String()))
+}
+
+func (enc *Encoder) setError(err os.Error) {
+	if enc.err == nil { // remember the first.
+		enc.err = err
+	}
+	enc.state.b.Reset()
+}
+
+// Send the data item preceded by a unsigned count of its length.
+func (enc *Encoder) send() {
+	// Encode the length.
+	enc.countState.encodeUint(uint64(enc.state.b.Len()))
+	// Build the buffer.
+	countLen := enc.countState.b.Len()
+	total := countLen + enc.state.b.Len()
+	if total > len(enc.buf) {
+		enc.buf = make([]byte, total+1000) // extra for growth
+	}
+	// Place the length before the data.
+	// TODO(r): avoid the extra copy here.
+	enc.countState.b.Read(enc.buf[0:countLen])
+	// Now the data.
+	enc.state.b.Read(enc.buf[countLen:total])
+	// Write the data.
+	_, err := enc.w.Write(enc.buf[0:total])
+	if err != nil {
+		enc.setError(err)
+	}
+}
+
+func (enc *Encoder) sendType(origt reflect.Type) (sent bool) {
+	// Drill down to the base type.
+	rt, _ := indirect(origt)
+
+	switch rt := rt.(type) {
+	default:
+		// Basic types and interfaces do not need to be described.
+		return
+	case *reflect.SliceType:
+		// If it's []uint8, don't send; it's considered basic.
+		if rt.Elem().Kind() == reflect.Uint8 {
+			return
+		}
+		// Otherwise we do send.
+		break
+	case *reflect.ArrayType:
+		// arrays must be sent so we know their lengths and element types.
+		break
+	case *reflect.MapType:
+		// maps must be sent so we know their lengths and key/value types.
+		break
+	case *reflect.StructType:
+		// structs must be sent so we know their fields.
+		break
+	case *reflect.ChanType, *reflect.FuncType:
+		// Probably a bad field in a struct.
+		enc.badType(rt)
+		return
+	}
+
+	// Have we already sent this type?  This time we ask about the base type.
+	if _, alreadySent := enc.sent[rt]; alreadySent {
+		return
+	}
+
+	// Need to send it.
+	typeLock.Lock()
+	info, err := getTypeInfo(rt)
+	typeLock.Unlock()
+	if err != nil {
+		enc.setError(err)
+		return
+	}
+	// Send the pair (-id, type)
+	// Id:
+	enc.state.encodeInt(-int64(info.id))
+	// Type:
+	enc.encode(enc.state.b, reflect.NewValue(info.wire))
+	enc.send()
+	if enc.err != nil {
+		return
+	}
+
+	// Remember we've sent this type.
+	enc.sent[rt] = info.id
+	// Remember we've sent the top-level, possibly indirect type too.
+	enc.sent[origt] = info.id
+	// Now send the inner types
+	switch st := rt.(type) {
+	case *reflect.StructType:
+		for i := 0; i < st.NumField(); i++ {
+			enc.sendType(st.Field(i).Type)
+		}
+	case reflect.ArrayOrSliceType:
+		enc.sendType(st.Elem())
+	}
+	return true
+}
+
+// Encode transmits the data item represented by the empty interface value,
+// guaranteeing that all necessary type information has been transmitted first.
+func (enc *Encoder) Encode(e interface{}) os.Error {
+	return enc.EncodeValue(reflect.NewValue(e))
+}
+
+// sendTypeId makes sure the remote side knows about this type.
+// It will send a descriptor if this is the first time the type has been
+// sent.  Regardless, it sends the id.
+func (enc *Encoder) sendTypeDescriptor(rt reflect.Type) {
+	// Make sure the type is known to the other side.
+	// First, have we already sent this type?
+	if _, alreadySent := enc.sent[rt]; !alreadySent {
+		// No, so send it.
+		sent := enc.sendType(rt)
+		if enc.err != nil {
+			return
+		}
+		// If the type info has still not been transmitted, it means we have
+		// a singleton basic type (int, []byte etc.) at top level.  We don't
+		// need to send the type info but we do need to update enc.sent.
+		if !sent {
+			typeLock.Lock()
+			info, err := getTypeInfo(rt)
+			typeLock.Unlock()
+			if err != nil {
+				enc.setError(err)
+				return
+			}
+			enc.sent[rt] = info.id
+		}
+	}
+
+	// Identify the type of this top-level value.
+	enc.state.encodeInt(int64(enc.sent[rt]))
+}
+
+// EncodeValue transmits the data item represented by the reflection value,
+// guaranteeing that all necessary type information has been transmitted first.
+func (enc *Encoder) EncodeValue(value reflect.Value) os.Error {
+	// Make sure we're single-threaded through here, so multiple
+	// goroutines can share an encoder.
+	enc.mutex.Lock()
+	defer enc.mutex.Unlock()
+
+	enc.err = nil
+	rt, _ := indirect(value.Type())
+
+	// Sanity check only: encoder should never come in with data present.
+	if enc.state.b.Len() > 0 || enc.countState.b.Len() > 0 {
+		enc.err = os.ErrorString("encoder: buffer not empty")
+		return enc.err
+	}
+
+	enc.sendTypeDescriptor(rt)
+	if enc.err != nil {
+		return enc.err
+	}
+
+	// Encode the object.
+	err := enc.encode(enc.state.b, value)
+	if err != nil {
+		enc.setError(err)
+	} else {
+		enc.send()
+	}
+
+	return enc.err
+}
diff --git a/libgo/go/gob/encoder_test.go b/libgo/go/gob/encoder_test.go
new file mode 100644
index 000000000..c2309352a
--- /dev/null
+++ b/libgo/go/gob/encoder_test.go
@@ -0,0 +1,385 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package gob
+
+import (
+	"bytes"
+	"io"
+	"os"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+type ET2 struct {
+	X string
+}
+
+type ET1 struct {
+	A    int
+	Et2  *ET2
+	Next *ET1
+}
+
+// Like ET1 but with a different name for a field
+type ET3 struct {
+	A             int
+	Et2           *ET2
+	DifferentNext *ET1
+}
+
+// Like ET1 but with a different type for a field
+type ET4 struct {
+	A    int
+	Et2  float64
+	Next int
+}
+
+func TestEncoderDecoder(t *testing.T) {
+	b := new(bytes.Buffer)
+	enc := NewEncoder(b)
+	et1 := new(ET1)
+	et1.A = 7
+	et1.Et2 = new(ET2)
+	err := enc.Encode(et1)
+	if err != nil {
+		t.Error("encoder fail:", err)
+	}
+	dec := NewDecoder(b)
+	newEt1 := new(ET1)
+	err = dec.Decode(newEt1)
+	if err != nil {
+		t.Fatal("error decoding ET1:", err)
+	}
+
+	if !reflect.DeepEqual(et1, newEt1) {
+		t.Fatalf("invalid data for et1: expected %+v; got %+v", *et1, *newEt1)
+	}
+	if b.Len() != 0 {
+		t.Error("not at eof;", b.Len(), "bytes left")
+	}
+
+	enc.Encode(et1)
+	newEt1 = new(ET1)
+	err = dec.Decode(newEt1)
+	if err != nil {
+		t.Fatal("round 2: error decoding ET1:", err)
+	}
+	if !reflect.DeepEqual(et1, newEt1) {
+		t.Fatalf("round 2: invalid data for et1: expected %+v; got %+v", *et1, *newEt1)
+	}
+	if b.Len() != 0 {
+		t.Error("round 2: not at eof;", b.Len(), "bytes left")
+	}
+
+	// Now test with a running encoder/decoder pair that we recognize a type mismatch.
+	err = enc.Encode(et1)
+	if err != nil {
+		t.Error("round 3: encoder fail:", err)
+	}
+	newEt2 := new(ET2)
+	err = dec.Decode(newEt2)
+	if err == nil {
+		t.Fatal("round 3: expected `bad type' error decoding ET2")
+	}
+}
+
+// Run one value through the encoder/decoder, but use the wrong type.
+// Input is always an ET1; we compare it to whatever is under 'e'.
+func badTypeCheck(e interface{}, shouldFail bool, msg string, t *testing.T) {
+	b := new(bytes.Buffer)
+	enc := NewEncoder(b)
+	et1 := new(ET1)
+	et1.A = 7
+	et1.Et2 = new(ET2)
+	err := enc.Encode(et1)
+	if err != nil {
+		t.Error("encoder fail:", err)
+	}
+	dec := NewDecoder(b)
+	err = dec.Decode(e)
+	if shouldFail && err == nil {
+		t.Error("expected error for", msg)
+	}
+	if !shouldFail && err != nil {
+		t.Error("unexpected error for", msg, err)
+	}
+}
+
+// Test that we recognize a bad type the first time.
+func TestWrongTypeDecoder(t *testing.T) {
+	badTypeCheck(new(ET2), true, "no fields in common", t)
+	badTypeCheck(new(ET3), false, "different name of field", t)
+	badTypeCheck(new(ET4), true, "different type of field", t)
+}
+
+func corruptDataCheck(s string, err os.Error, t *testing.T) {
+	b := bytes.NewBufferString(s)
+	dec := NewDecoder(b)
+	err1 := dec.Decode(new(ET2))
+	if err1 != err {
+		t.Error("expected error", err, "got", err1)
+	}
+}
+
+// Check that we survive bad data.
+func TestBadData(t *testing.T) {
+	corruptDataCheck("", os.EOF, t)
+	corruptDataCheck("\x7Fhi", io.ErrUnexpectedEOF, t)
+	corruptDataCheck("\x03now is the time for all good men", errBadType, t)
+}
+
+// Types not supported by the Encoder.
+var unsupportedValues = []interface{}{
+	make(chan int),
+	func(a int) bool { return true },
+}
+
+func TestUnsupported(t *testing.T) {
+	var b bytes.Buffer
+	enc := NewEncoder(&b)
+	for _, v := range unsupportedValues {
+		err := enc.Encode(v)
+		if err == nil {
+			t.Errorf("expected error for %T; got none", v)
+		}
+	}
+}
+
+func encAndDec(in, out interface{}) os.Error {
+	b := new(bytes.Buffer)
+	enc := NewEncoder(b)
+	err := enc.Encode(in)
+	if err != nil {
+		return err
+	}
+	dec := NewDecoder(b)
+	err = dec.Decode(out)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func TestTypeToPtrType(t *testing.T) {
+	// Encode a T, decode a *T
+	type Type0 struct {
+		A int
+	}
+	t0 := Type0{7}
+	t0p := (*Type0)(nil)
+	if err := encAndDec(t0, t0p); err != nil {
+		t.Error(err)
+	}
+}
+
+func TestPtrTypeToType(t *testing.T) {
+	// Encode a *T, decode a T
+	type Type1 struct {
+		A uint
+	}
+	t1p := &Type1{17}
+	var t1 Type1
+	if err := encAndDec(t1, t1p); err != nil {
+		t.Error(err)
+	}
+}
+
+func TestTypeToPtrPtrPtrPtrType(t *testing.T) {
+	type Type2 struct {
+		A ****float64
+	}
+	t2 := Type2{}
+	t2.A = new(***float64)
+	*t2.A = new(**float64)
+	**t2.A = new(*float64)
+	***t2.A = new(float64)
+	****t2.A = 27.4
+	t2pppp := new(***Type2)
+	if err := encAndDec(t2, t2pppp); err != nil {
+		t.Fatal(err)
+	}
+	if ****(****t2pppp).A != ****t2.A {
+		t.Errorf("wrong value after decode: %g not %g", ****(****t2pppp).A, ****t2.A)
+	}
+}
+
+func TestSlice(t *testing.T) {
+	type Type3 struct {
+		A []string
+	}
+	t3p := &Type3{[]string{"hello", "world"}}
+	var t3 Type3
+	if err := encAndDec(t3, t3p); err != nil {
+		t.Error(err)
+	}
+}
+
+func TestValueError(t *testing.T) {
+	// Encode a *T, decode a T
+	type Type4 struct {
+		a int
+	}
+	t4p := &Type4{3}
+	var t4 Type4 // note: not a pointer.
+	if err := encAndDec(t4p, t4); err == nil || strings.Index(err.String(), "pointer") < 0 {
+		t.Error("expected error about pointer; got", err)
+	}
+}
+
+func TestArray(t *testing.T) {
+	type Type5 struct {
+		A [3]string
+		B [3]byte
+	}
+	type Type6 struct {
+		A [2]string // can't hold t5.a
+	}
+	t5 := Type5{[3]string{"hello", ",", "world"}, [3]byte{1, 2, 3}}
+	var t5p Type5
+	if err := encAndDec(t5, &t5p); err != nil {
+		t.Error(err)
+	}
+	var t6 Type6
+	if err := encAndDec(t5, &t6); err == nil {
+		t.Error("should fail with mismatched array sizes")
+	}
+}
+
+// Regression test for bug: must send zero values inside arrays
+func TestDefaultsInArray(t *testing.T) {
+	type Type7 struct {
+		B []bool
+		I []int
+		S []string
+		F []float64
+	}
+	t7 := Type7{
+		[]bool{false, false, true},
+		[]int{0, 0, 1},
+		[]string{"hi", "", "there"},
+		[]float64{0, 0, 1},
+	}
+	var t7p Type7
+	if err := encAndDec(t7, &t7p); err != nil {
+		t.Error(err)
+	}
+}
+
+var testInt int
+var testFloat32 float32
+var testString string
+var testSlice []string
+var testMap map[string]int
+var testArray [7]int
+
+type SingleTest struct {
+	in  interface{}
+	out interface{}
+	err string
+}
+
+var singleTests = []SingleTest{
+	{17, &testInt, ""},
+	{float32(17.5), &testFloat32, ""},
+	{"bike shed", &testString, ""},
+	{[]string{"bike", "shed", "paint", "color"}, &testSlice, ""},
+	{map[string]int{"seven": 7, "twelve": 12}, &testMap, ""},
+	{[7]int{4, 55, 0, 0, 0, 0, 0}, &testArray, ""}, // case that once triggered a bug
+	{[7]int{4, 55, 1, 44, 22, 66, 1234}, &testArray, ""},
+
+	// Decode errors
+	{172, &testFloat32, "wrong type"},
+}
+
+func TestSingletons(t *testing.T) {
+	b := new(bytes.Buffer)
+	enc := NewEncoder(b)
+	dec := NewDecoder(b)
+	for _, test := range singleTests {
+		b.Reset()
+		err := enc.Encode(test.in)
+		if err != nil {
+			t.Errorf("error encoding %v: %s", test.in, err)
+			continue
+		}
+		err = dec.Decode(test.out)
+		switch {
+		case err != nil && test.err == "":
+			t.Errorf("error decoding %v: %s", test.in, err)
+			continue
+		case err == nil && test.err != "":
+			t.Errorf("expected error decoding %v: %s", test.in, test.err)
+			continue
+		case err != nil && test.err != "":
+			if strings.Index(err.String(), test.err) < 0 {
+				t.Errorf("wrong error decoding %v: wanted %s, got %v", test.in, test.err, err)
+			}
+			continue
+		}
+		// Get rid of the pointer in the rhs
+		val := reflect.NewValue(test.out).(*reflect.PtrValue).Elem().Interface()
+		if !reflect.DeepEqual(test.in, val) {
+			t.Errorf("decoding singleton: expected %v got %v", test.in, val)
+		}
+	}
+}
+
+func TestStructNonStruct(t *testing.T) {
+	type Struct struct {
+		A string
+	}
+	type NonStruct string
+	s := Struct{"hello"}
+	var sp Struct
+	if err := encAndDec(s, &sp); err != nil {
+		t.Error(err)
+	}
+	var ns NonStruct
+	if err := encAndDec(s, &ns); err == nil {
+		t.Error("should get error for struct/non-struct")
+	} else if strings.Index(err.String(), "type") < 0 {
+		t.Error("for struct/non-struct expected type error; got", err)
+	}
+	// Now try the other way
+	var nsp NonStruct
+	if err := encAndDec(ns, &nsp); err != nil {
+		t.Error(err)
+	}
+	if err := encAndDec(ns, &s); err == nil {
+		t.Error("should get error for non-struct/struct")
+	} else if strings.Index(err.String(), "type") < 0 {
+		t.Error("for non-struct/struct expected type error; got", err)
+	}
+}
+
+type interfaceIndirectTestI interface {
+	F() bool
+}
+
+type interfaceIndirectTestT struct{}
+
+func (this *interfaceIndirectTestT) F() bool {
+	return true
+}
+
+// A version of a bug reported on golang-nuts.  Also tests top-level
+// slice of interfaces.  The issue was registering *T caused T to be
+// stored as the concrete type.
+func TestInterfaceIndirect(t *testing.T) {
+	Register(&interfaceIndirectTestT{})
+	b := new(bytes.Buffer)
+	w := []interfaceIndirectTestI{&interfaceIndirectTestT{}}
+	err := NewEncoder(b).Encode(w)
+	if err != nil {
+		t.Fatal("encode error:", err)
+	}
+
+	var r []interfaceIndirectTestI
+	err = NewDecoder(b).Decode(&r)
+	if err != nil {
+		t.Fatal("decode error:", err)
+	}
+}
diff --git a/libgo/go/gob/error.go b/libgo/go/gob/error.go
new file mode 100644
index 000000000..b053761fb
--- /dev/null
+++ b/libgo/go/gob/error.go
@@ -0,0 +1,41 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package gob
+
+import (
+	"fmt"
+	"os"
+)
+
+// Errors in decoding and encoding are handled using panic and recover.
+// Panics caused by user error (that is, everything except run-time panics
+// such as "index out of bounds" errors) do not leave the file that caused
+// them, but are instead turned into plain os.Error returns.  Encoding and
+// decoding functions and methods that do not return an os.Error either use
+// panic to report an error or are guaranteed error-free.
+
+// A gobError wraps an os.Error and is used to distinguish errors (panics) generated in this package.
+type gobError struct {
+	os.Error
+}
+
+// errorf is like error but takes Printf-style arguments to construct an os.Error.
+func errorf(format string, args ...interface{}) {
+	error(fmt.Errorf(format, args...))
+}
+
+// error wraps the argument error and uses it as the argument to panic.
+func error(err os.Error) {
+	panic(gobError{Error: err})
+}
+
+// catchError is meant to be used as a deferred function to turn a panic(gobError) into a
+// plain os.Error.  It overwrites the error return of the function that deferred its call.
+func catchError(err *os.Error) {
+	if e := recover(); e != nil {
+		*err = e.(gobError).Error // Will re-panic if not one of our errors, such as a runtime error.
+	}
+	return
+}
diff --git a/libgo/go/gob/type.go b/libgo/go/gob/type.go
new file mode 100644
index 000000000..22502a6e6
--- /dev/null
+++ b/libgo/go/gob/type.go
@@ -0,0 +1,539 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package gob
+
+import (
+	"fmt"
+	"os"
+	"reflect"
+	"sync"
+)
+
+// Reflection types are themselves interface values holding structs
+// describing the type.  Each type has a different struct so that struct can
+// be the kind.  For example, if typ is the reflect type for an int8, typ is
+// a pointer to a reflect.Int8Type struct; if typ is the reflect type for a
+// function, typ is a pointer to a reflect.FuncType struct; we use the type
+// of that pointer as the kind.
+
+// A typeId represents a gob Type as an integer that can be passed on the wire.
+// Internally, typeIds are used as keys to a map to recover the underlying type info.
+type typeId int32
+
+var nextId typeId       // incremented for each new type we build
+var typeLock sync.Mutex // set while building a type
+const firstUserId = 64  // lowest id number granted to user
+
+type gobType interface {
+	id() typeId
+	setId(id typeId)
+	name() string
+	string() string // not public; only for debugging
+	safeString(seen map[typeId]bool) string
+}
+
+var types = make(map[reflect.Type]gobType)
+var idToType = make(map[typeId]gobType)
+var builtinIdToType map[typeId]gobType // set in init() after builtins are established
+
+func setTypeId(typ gobType) {
+	nextId++
+	typ.setId(nextId)
+	idToType[nextId] = typ
+}
+
+func (t typeId) gobType() gobType {
+	if t == 0 {
+		return nil
+	}
+	return idToType[t]
+}
+
+// string returns the string representation of the type associated with the typeId.
+func (t typeId) string() string {
+	if t.gobType() == nil {
+		return ""
+	}
+	return t.gobType().string()
+}
+
+// Name returns the name of the type associated with the typeId.
+func (t typeId) name() string {
+	if t.gobType() == nil {
+		return ""
+	}
+	return t.gobType().name()
+}
+
+// Common elements of all types.
+type CommonType struct {
+	Name string
+	Id   typeId
+}
+
+func (t *CommonType) id() typeId { return t.Id }
+
+func (t *CommonType) setId(id typeId) { t.Id = id }
+
+func (t *CommonType) string() string { return t.Name }
+
+func (t *CommonType) safeString(seen map[typeId]bool) string {
+	return t.Name
+}
+
+func (t *CommonType) name() string { return t.Name }
+
+// Create and check predefined types
+// The string for tBytes is "bytes" not "[]byte" to signify its specialness.
+
+var (
+	// Primordial types, needed during initialization.
+	tBool      = bootstrapType("bool", false, 1)
+	tInt       = bootstrapType("int", int(0), 2)
+	tUint      = bootstrapType("uint", uint(0), 3)
+	tFloat     = bootstrapType("float", 0.0, 4)
+	tBytes     = bootstrapType("bytes", make([]byte, 0), 5)
+	tString    = bootstrapType("string", "", 6)
+	tComplex   = bootstrapType("complex", 0+0i, 7)
+	tInterface = bootstrapType("interface", interface{}(nil), 8)
+	// Reserve some Ids for compatible expansion
+	tReserved7 = bootstrapType("_reserved1", struct{ r7 int }{}, 9)
+	tReserved6 = bootstrapType("_reserved1", struct{ r6 int }{}, 10)
+	tReserved5 = bootstrapType("_reserved1", struct{ r5 int }{}, 11)
+	tReserved4 = bootstrapType("_reserved1", struct{ r4 int }{}, 12)
+	tReserved3 = bootstrapType("_reserved1", struct{ r3 int }{}, 13)
+	tReserved2 = bootstrapType("_reserved1", struct{ r2 int }{}, 14)
+	tReserved1 = bootstrapType("_reserved1", struct{ r1 int }{}, 15)
+)
+
+// Predefined because it's needed by the Decoder
+var tWireType = mustGetTypeInfo(reflect.Typeof(wireType{})).id
+
+func init() {
+	// Some magic numbers to make sure there are no surprises.
+	checkId(16, tWireType)
+	checkId(17, mustGetTypeInfo(reflect.Typeof(arrayType{})).id)
+	checkId(18, mustGetTypeInfo(reflect.Typeof(CommonType{})).id)
+	checkId(19, mustGetTypeInfo(reflect.Typeof(sliceType{})).id)
+	checkId(20, mustGetTypeInfo(reflect.Typeof(structType{})).id)
+	checkId(21, mustGetTypeInfo(reflect.Typeof(fieldType{})).id)
+	checkId(23, mustGetTypeInfo(reflect.Typeof(mapType{})).id)
+
+	builtinIdToType = make(map[typeId]gobType)
+	for k, v := range idToType {
+		builtinIdToType[k] = v
+	}
+
+	// Move the id space upwards to allow for growth in the predefined world
+	// without breaking existing files.
+	if nextId > firstUserId {
+		panic(fmt.Sprintln("nextId too large:", nextId))
+	}
+	nextId = firstUserId
+	registerBasics()
+}
+
+// Array type
+type arrayType struct {
+	CommonType
+	Elem typeId
+	Len  int
+}
+
+func newArrayType(name string, elem gobType, length int) *arrayType {
+	a := &arrayType{CommonType{Name: name}, elem.id(), length}
+	setTypeId(a)
+	return a
+}
+
+func (a *arrayType) safeString(seen map[typeId]bool) string {
+	if seen[a.Id] {
+		return a.Name
+	}
+	seen[a.Id] = true
+	return fmt.Sprintf("[%d]%s", a.Len, a.Elem.gobType().safeString(seen))
+}
+
+func (a *arrayType) string() string { return a.safeString(make(map[typeId]bool)) }
+
+// Map type
+type mapType struct {
+	CommonType
+	Key  typeId
+	Elem typeId
+}
+
+func newMapType(name string, key, elem gobType) *mapType {
+	m := &mapType{CommonType{Name: name}, key.id(), elem.id()}
+	setTypeId(m)
+	return m
+}
+
+func (m *mapType) safeString(seen map[typeId]bool) string {
+	if seen[m.Id] {
+		return m.Name
+	}
+	seen[m.Id] = true
+	key := m.Key.gobType().safeString(seen)
+	elem := m.Elem.gobType().safeString(seen)
+	return fmt.Sprintf("map[%s]%s", key, elem)
+}
+
+func (m *mapType) string() string { return m.safeString(make(map[typeId]bool)) }
+
+// Slice type
+type sliceType struct {
+	CommonType
+	Elem typeId
+}
+
+func newSliceType(name string, elem gobType) *sliceType {
+	s := &sliceType{CommonType{Name: name}, elem.id()}
+	setTypeId(s)
+	return s
+}
+
+func (s *sliceType) safeString(seen map[typeId]bool) string {
+	if seen[s.Id] {
+		return s.Name
+	}
+	seen[s.Id] = true
+	return fmt.Sprintf("[]%s", s.Elem.gobType().safeString(seen))
+}
+
+func (s *sliceType) string() string { return s.safeString(make(map[typeId]bool)) }
+
+// Struct type
+type fieldType struct {
+	Name string
+	Id   typeId
+}
+
+type structType struct {
+	CommonType
+	Field []*fieldType
+}
+
+func (s *structType) safeString(seen map[typeId]bool) string {
+	if s == nil {
+		return ""
+	}
+	if _, ok := seen[s.Id]; ok {
+		return s.Name
+	}
+	seen[s.Id] = true
+	str := s.Name + " = struct { "
+	for _, f := range s.Field {
+		str += fmt.Sprintf("%s %s; ", f.Name, f.Id.gobType().safeString(seen))
+	}
+	str += "}"
+	return str
+}
+
+func (s *structType) string() string { return s.safeString(make(map[typeId]bool)) }
+
+func newStructType(name string) *structType {
+	s := &structType{CommonType{Name: name}, nil}
+	setTypeId(s)
+	return s
+}
+
+// Step through the indirections on a type to discover the base type.
+// Return the base type and the number of indirections.
+func indirect(t reflect.Type) (rt reflect.Type, count int) {
+	rt = t
+	for {
+		pt, ok := rt.(*reflect.PtrType)
+		if !ok {
+			break
+		}
+		rt = pt.Elem()
+		count++
+	}
+	return
+}
+
+func newTypeObject(name string, rt reflect.Type) (gobType, os.Error) {
+	switch t := rt.(type) {
+	// All basic types are easy: they are predefined.
+	case *reflect.BoolType:
+		return tBool.gobType(), nil
+
+	case *reflect.IntType:
+		return tInt.gobType(), nil
+
+	case *reflect.UintType:
+		return tUint.gobType(), nil
+
+	case *reflect.FloatType:
+		return tFloat.gobType(), nil
+
+	case *reflect.ComplexType:
+		return tComplex.gobType(), nil
+
+	case *reflect.StringType:
+		return tString.gobType(), nil
+
+	case *reflect.InterfaceType:
+		return tInterface.gobType(), nil
+
+	case *reflect.ArrayType:
+		gt, err := getType("", t.Elem())
+		if err != nil {
+			return nil, err
+		}
+		return newArrayType(name, gt, t.Len()), nil
+
+	case *reflect.MapType:
+		kt, err := getType("", t.Key())
+		if err != nil {
+			return nil, err
+		}
+		vt, err := getType("", t.Elem())
+		if err != nil {
+			return nil, err
+		}
+		return newMapType(name, kt, vt), nil
+
+	case *reflect.SliceType:
+		// []byte == []uint8 is a special case
+		if t.Elem().Kind() == reflect.Uint8 {
+			return tBytes.gobType(), nil
+		}
+		gt, err := getType(t.Elem().Name(), t.Elem())
+		if err != nil {
+			return nil, err
+		}
+		return newSliceType(name, gt), nil
+
+	case *reflect.StructType:
+		// Install the struct type itself before the fields so recursive
+		// structures can be constructed safely.
+		strType := newStructType(name)
+		types[rt] = strType
+		idToType[strType.id()] = strType
+		field := make([]*fieldType, t.NumField())
+		for i := 0; i < t.NumField(); i++ {
+			f := t.Field(i)
+			typ, _ := indirect(f.Type)
+			tname := typ.Name()
+			if tname == "" {
+				t, _ := indirect(f.Type)
+				tname = t.String()
+			}
+			gt, err := getType(tname, f.Type)
+			if err != nil {
+				return nil, err
+			}
+			field[i] = &fieldType{f.Name, gt.id()}
+		}
+		strType.Field = field
+		return strType, nil
+
+	default:
+		return nil, os.ErrorString("gob NewTypeObject can't handle type: " + rt.String())
+	}
+	return nil, nil
+}
+
+// getType returns the Gob type describing the given reflect.Type.
+// typeLock must be held.
+func getType(name string, rt reflect.Type) (gobType, os.Error) {
+	rt, _ = indirect(rt)
+	typ, present := types[rt]
+	if present {
+		return typ, nil
+	}
+	typ, err := newTypeObject(name, rt)
+	if err == nil {
+		types[rt] = typ
+	}
+	return typ, err
+}
+
+func checkId(want, got typeId) {
+	if want != got {
+		fmt.Fprintf(os.Stderr, "checkId: %d should be %d\n", int(want), int(got))
+		panic("bootstrap type wrong id: " + got.name() + " " + got.string() + " not " + want.string())
+	}
+}
+
+// used for building the basic types; called only from init()
+func bootstrapType(name string, e interface{}, expect typeId) typeId {
+	rt := reflect.Typeof(e)
+	_, present := types[rt]
+	if present {
+		panic("bootstrap type already present: " + name + ", " + rt.String())
+	}
+	typ := &CommonType{Name: name}
+	types[rt] = typ
+	setTypeId(typ)
+	checkId(expect, nextId)
+	return nextId
+}
+
+// Representation of the information we send and receive about this type.
+// Each value we send is preceded by its type definition: an encoded int.
+// However, the very first time we send the value, we first send the pair
+// (-id, wireType).
+// For bootstrapping purposes, we assume that the recipient knows how
+// to decode a wireType; it is exactly the wireType struct here, interpreted
+// using the gob rules for sending a structure, except that we assume the
+// ids for wireType and structType are known.  The relevant pieces
+// are built in encode.go's init() function.
+// To maintain binary compatibility, if you extend this type, always put
+// the new fields last.
+type wireType struct {
+	ArrayT  *arrayType
+	SliceT  *sliceType
+	StructT *structType
+	MapT    *mapType
+}
+
+func (w *wireType) string() string {
+	const unknown = "unknown type"
+	if w == nil {
+		return unknown
+	}
+	switch {
+	case w.ArrayT != nil:
+		return w.ArrayT.Name
+	case w.SliceT != nil:
+		return w.SliceT.Name
+	case w.StructT != nil:
+		return w.StructT.Name
+	case w.MapT != nil:
+		return w.MapT.Name
+	}
+	return unknown
+}
+
+type typeInfo struct {
+	id      typeId
+	encoder *encEngine
+	wire    *wireType
+}
+
+var typeInfoMap = make(map[reflect.Type]*typeInfo) // protected by typeLock
+
+// The reflection type must have all its indirections processed out.
+// typeLock must be held.
+func getTypeInfo(rt reflect.Type) (*typeInfo, os.Error) {
+	if rt.Kind() == reflect.Ptr {
+		panic("pointer type in getTypeInfo: " + rt.String())
+	}
+	info, ok := typeInfoMap[rt]
+	if !ok {
+		info = new(typeInfo)
+		name := rt.Name()
+		gt, err := getType(name, rt)
+		if err != nil {
+			return nil, err
+		}
+		info.id = gt.id()
+		t := info.id.gobType()
+		switch typ := rt.(type) {
+		case *reflect.ArrayType:
+			info.wire = &wireType{ArrayT: t.(*arrayType)}
+		case *reflect.MapType:
+			info.wire = &wireType{MapT: t.(*mapType)}
+		case *reflect.SliceType:
+			// []byte == []uint8 is a special case handled separately
+			if typ.Elem().Kind() != reflect.Uint8 {
+				info.wire = &wireType{SliceT: t.(*sliceType)}
+			}
+		case *reflect.StructType:
+			info.wire = &wireType{StructT: t.(*structType)}
+		}
+		typeInfoMap[rt] = info
+	}
+	return info, nil
+}
+
+// Called only when a panic is acceptable and unexpected.
+func mustGetTypeInfo(rt reflect.Type) *typeInfo {
+	t, err := getTypeInfo(rt)
+	if err != nil {
+		panic("getTypeInfo: " + err.String())
+	}
+	return t
+}
+
+var (
+	nameToConcreteType = make(map[string]reflect.Type)
+	concreteTypeToName = make(map[reflect.Type]string)
+)
+
+// RegisterName is like Register but uses the provided name rather than the
+// type's default.
+func RegisterName(name string, value interface{}) {
+	if name == "" {
+		// reserved for nil
+		panic("attempt to register empty name")
+	}
+	rt, _ := indirect(reflect.Typeof(value))
+	// Check for incompatible duplicates.
+	if t, ok := nameToConcreteType[name]; ok && t != rt {
+		panic("gob: registering duplicate types for " + name)
+	}
+	if n, ok := concreteTypeToName[rt]; ok && n != name {
+		panic("gob: registering duplicate names for " + rt.String())
+	}
+	// Store the name and type provided by the user....
+	nameToConcreteType[name] = reflect.Typeof(value)
+	// but the flattened type in the type table, since that's what decode needs.
+	concreteTypeToName[rt] = name
+}
+
+// Register records a type, identified by a value for that type, under its
+// internal type name.  That name will identify the concrete type of a value
+// sent or received as an interface variable.  Only types that will be
+// transferred as implementations of interface values need to be registered.
+// Expecting to be used only during initialization, it panics if the mapping
+// between types and names is not a bijection.
+func Register(value interface{}) {
+	// Default to printed representation for unnamed types
+	rt := reflect.Typeof(value)
+	name := rt.String()
+
+	// But for named types (or pointers to them), qualify with import path.
+	// Dereference one pointer looking for a named type.
+	star := ""
+	if rt.Name() == "" {
+		if pt, ok := rt.(*reflect.PtrType); ok {
+			star = "*"
+			rt = pt
+		}
+	}
+	if rt.Name() != "" {
+		if rt.PkgPath() == "" {
+			name = star + rt.Name()
+		} else {
+			name = star + rt.PkgPath() + "." + rt.Name()
+		}
+	}
+
+	RegisterName(name, value)
+}
+
+func registerBasics() {
+	Register(int(0))
+	Register(int8(0))
+	Register(int16(0))
+	Register(int32(0))
+	Register(int64(0))
+	Register(uint(0))
+	Register(uint8(0))
+	Register(uint16(0))
+	Register(uint32(0))
+	Register(uint64(0))
+	Register(float32(0))
+	Register(0.0)
+	Register(complex64(0i))
+	Register(complex128(0i))
+	Register(false)
+	Register("")
+	Register([]byte(nil))
+}
diff --git a/libgo/go/gob/type_test.go b/libgo/go/gob/type_test.go
new file mode 100644
index 000000000..5aecde103
--- /dev/null
+++ b/libgo/go/gob/type_test.go
@@ -0,0 +1,153 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package gob
+
+import (
+	"reflect"
+	"testing"
+)
+
+type typeT struct {
+	id  typeId
+	str string
+}
+
+var basicTypes = []typeT{
+	{tBool, "bool"},
+	{tInt, "int"},
+	{tUint, "uint"},
+	{tFloat, "float"},
+	{tBytes, "bytes"},
+	{tString, "string"},
+}
+
+func getTypeUnlocked(name string, rt reflect.Type) gobType {
+	typeLock.Lock()
+	defer typeLock.Unlock()
+	t, err := getType(name, rt)
+	if err != nil {
+		panic("getTypeUnlocked: " + err.String())
+	}
+	return t
+}
+
+// Sanity checks
+func TestBasic(t *testing.T) {
+	for _, tt := range basicTypes {
+		if tt.id.string() != tt.str {
+			t.Errorf("checkType: expected %q got %s", tt.str, tt.id.string())
+		}
+		if tt.id == 0 {
+			t.Errorf("id for %q is zero", tt.str)
+		}
+	}
+}
+
+// Reregister some basic types to check registration is idempotent.
+func TestReregistration(t *testing.T) {
+	newtyp := getTypeUnlocked("int", reflect.Typeof(int(0)))
+	if newtyp != tInt.gobType() {
+		t.Errorf("reregistration of %s got new type", newtyp.string())
+	}
+	newtyp = getTypeUnlocked("uint", reflect.Typeof(uint(0)))
+	if newtyp != tUint.gobType() {
+		t.Errorf("reregistration of %s got new type", newtyp.string())
+	}
+	newtyp = getTypeUnlocked("string", reflect.Typeof("hello"))
+	if newtyp != tString.gobType() {
+		t.Errorf("reregistration of %s got new type", newtyp.string())
+	}
+}
+
+func TestArrayType(t *testing.T) {
+	var a3 [3]int
+	a3int := getTypeUnlocked("foo", reflect.Typeof(a3))
+	newa3int := getTypeUnlocked("bar", reflect.Typeof(a3))
+	if a3int != newa3int {
+		t.Errorf("second registration of [3]int creates new type")
+	}
+	var a4 [4]int
+	a4int := getTypeUnlocked("goo", reflect.Typeof(a4))
+	if a3int == a4int {
+		t.Errorf("registration of [3]int creates same type as [4]int")
+	}
+	var b3 [3]bool
+	a3bool := getTypeUnlocked("", reflect.Typeof(b3))
+	if a3int == a3bool {
+		t.Errorf("registration of [3]bool creates same type as [3]int")
+	}
+	str := a3bool.string()
+	expected := "[3]bool"
+	if str != expected {
+		t.Errorf("array printed as %q; expected %q", str, expected)
+	}
+}
+
+func TestSliceType(t *testing.T) {
+	var s []int
+	sint := getTypeUnlocked("slice", reflect.Typeof(s))
+	var news []int
+	newsint := getTypeUnlocked("slice1", reflect.Typeof(news))
+	if sint != newsint {
+		t.Errorf("second registration of []int creates new type")
+	}
+	var b []bool
+	sbool := getTypeUnlocked("", reflect.Typeof(b))
+	if sbool == sint {
+		t.Errorf("registration of []bool creates same type as []int")
+	}
+	str := sbool.string()
+	expected := "[]bool"
+	if str != expected {
+		t.Errorf("slice printed as %q; expected %q", str, expected)
+	}
+}
+
+func TestMapType(t *testing.T) {
+	var m map[string]int
+	mapStringInt := getTypeUnlocked("map", reflect.Typeof(m))
+	var newm map[string]int
+	newMapStringInt := getTypeUnlocked("map1", reflect.Typeof(newm))
+	if mapStringInt != newMapStringInt {
+		t.Errorf("second registration of map[string]int creates new type")
+	}
+	var b map[string]bool
+	mapStringBool := getTypeUnlocked("", reflect.Typeof(b))
+	if mapStringBool == mapStringInt {
+		t.Errorf("registration of map[string]bool creates same type as map[string]int")
+	}
+	str := mapStringBool.string()
+	expected := "map[string]bool"
+	if str != expected {
+		t.Errorf("map printed as %q; expected %q", str, expected)
+	}
+}
+
+type Bar struct {
+	x string
+}
+
+// This structure has pointers and refers to itself, making it a good test case.
+type Foo struct {
+	a int
+	b int32 // will become int
+	c string
+	d []byte
+	e *float64    // will become float64
+	f ****float64 // will become float64
+	g *Bar
+	h *Bar // should not interpolate the definition of Bar again
+	i *Foo // will not explode
+}
+
+func TestStructType(t *testing.T) {
+	sstruct := getTypeUnlocked("Foo", reflect.Typeof(Foo{}))
+	str := sstruct.string()
+	// If we can print it correctly, we built it correctly.
+	expected := "Foo = struct { a int; b int; c string; d bytes; e float; f float; g Bar = struct { x string; }; h Bar; i Foo; }"
+	if str != expected {
+		t.Errorf("struct printed as %q; expected %q", str, expected)
+	}
+}
diff --git a/libgo/go/hash/adler32/adler32.go b/libgo/go/hash/adler32/adler32.go
new file mode 100644
index 000000000..cd0c2599a
--- /dev/null
+++ b/libgo/go/hash/adler32/adler32.go
@@ -0,0 +1,88 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This package implements the Adler-32 checksum.
+// Defined in RFC 1950:
+//	Adler-32 is composed of two sums accumulated per byte: s1 is
+//	the sum of all bytes, s2 is the sum of all s1 values. Both sums
+//	are done modulo 65521. s1 is initialized to 1, s2 to zero.  The
+//	Adler-32 checksum is stored as s2*65536 + s1 in most-
+//	significant-byte first (network) order.
+package adler32
+
+import (
+	"hash"
+	"os"
+)
+
+const (
+	mod = 65521
+)
+
+// The size of an Adler-32 checksum in bytes.
+const Size = 4
+
+// digest represents the partial evaluation of a checksum.
+type digest struct {
+	// invariant: (a < mod && b < mod) || a <= b
+	// invariant: a + b + 255 <= 0xffffffff
+	a, b uint32
+}
+
+func (d *digest) Reset() { d.a, d.b = 1, 0 }
+
+// New returns a new hash.Hash32 computing the Adler-32 checksum.
+func New() hash.Hash32 {
+	d := new(digest)
+	d.Reset()
+	return d
+}
+
+func (d *digest) Size() int { return Size }
+
+// Add p to the running checksum a, b.
+func update(a, b uint32, p []byte) (aa, bb uint32) {
+	for i := 0; i < len(p); i++ {
+		a += uint32(p[i])
+		b += a
+		// invariant: a <= b
+		if b > (0xffffffff-255)/2 {
+			a %= mod
+			b %= mod
+			// invariant: a < mod && b < mod
+		} else {
+			// invariant: a + b + 255 <= 2 * b + 255 <= 0xffffffff
+		}
+	}
+	return a, b
+}
+
+// Return the 32-bit checksum corresponding to a, b.
+func finish(a, b uint32) uint32 {
+	if b >= mod {
+		a %= mod
+		b %= mod
+	}
+	return b<<16 | a
+}
+
+func (d *digest) Write(p []byte) (nn int, err os.Error) {
+	d.a, d.b = update(d.a, d.b, p)
+	return len(p), nil
+}
+
+func (d *digest) Sum32() uint32 { return finish(d.a, d.b) }
+
+func (d *digest) Sum() []byte {
+	p := make([]byte, 4)
+	s := d.Sum32()
+	p[0] = byte(s >> 24)
+	p[1] = byte(s >> 16)
+	p[2] = byte(s >> 8)
+	p[3] = byte(s)
+	return p
+}
+
+// Checksum returns the Adler-32 checksum of data.
+func Checksum(data []byte) uint32 { return finish(update(1, 0, data)) }
diff --git a/libgo/go/hash/adler32/adler32_test.go b/libgo/go/hash/adler32/adler32_test.go
new file mode 100644
index 000000000..ffa5569bc
--- /dev/null
+++ b/libgo/go/hash/adler32/adler32_test.go
@@ -0,0 +1,63 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package adler32
+
+import (
+	"io"
+	"testing"
+)
+
+type _Adler32Test struct {
+	out uint32
+	in  string
+}
+
+var golden = []_Adler32Test{
+	{0x1, ""},
+	{0x620062, "a"},
+	{0x12600c4, "ab"},
+	{0x24d0127, "abc"},
+	{0x3d8018b, "abcd"},
+	{0x5c801f0, "abcde"},
+	{0x81e0256, "abcdef"},
+	{0xadb02bd, "abcdefg"},
+	{0xe000325, "abcdefgh"},
+	{0x118e038e, "abcdefghi"},
+	{0x158603f8, "abcdefghij"},
+	{0x3f090f02, "Discard medicine more than two years old."},
+	{0x46d81477, "He who has a shady past knows that nice guys finish last."},
+	{0x40ee0ee1, "I wouldn't marry him with a ten foot pole."},
+	{0x16661315, "Free! Free!/A trip/to Mars/for 900/empty jars/Burma Shave"},
+	{0x5b2e1480, "The days of the digital watch are numbered.  -Tom Stoppard"},
+	{0x8c3c09ea, "Nepal premier won't resign."},
+	{0x45ac18fd, "For every action there is an equal and opposite government program."},
+	{0x53c61462, "His money is twice tainted: 'taint yours and 'taint mine."},
+	{0x7e511e63, "There is no reason for any individual to have a computer in their home. -Ken Olsen, 1977"},
+	{0xe4801a6a, "It's a tiny change to the code and not completely disgusting. - Bob Manchek"},
+	{0x61b507df, "size:  a.out:  bad magic"},
+	{0xb8631171, "The major problem is with sendmail.  -Mark Horton"},
+	{0x8b5e1904, "Give me a rock, paper and scissors and I will move the world.  CCFestoon"},
+	{0x7cc6102b, "If the enemy is within range, then so are you."},
+	{0x700318e7, "It's well we cannot hear the screams/That we create in others' dreams."},
+	{0x1e601747, "You remind me of a TV show, but that's all right: I watch it anyway."},
+	{0xb55b0b09, "C is as portable as Stonehedge!!"},
+	{0x39111dd0, "Even if I could be Shakespeare, I think I should still choose to be Faraday. - A. Huxley"},
+	{0x91dd304f, "The fugacity of a constituent in a mixture of gases at a given temperature is proportional to its mole fraction.  Lewis-Randall Rule"},
+	{0x2e5d1316, "How can you write a big system without C++?  -Paul Glick"},
+	{0xd0201df6, "'Invariant assertions' is the most elegant programming technique!  -Tom Szymanski"},
+}
+
+func TestGolden(t *testing.T) {
+	for i := 0; i < len(golden); i++ {
+		g := golden[i]
+		c := New()
+		io.WriteString(c, g.in)
+		s := c.Sum32()
+		if s != g.out {
+			t.Errorf("adler32(%s) = 0x%x want 0x%x", g.in, s, g.out)
+			t.FailNow()
+		}
+	}
+}
diff --git a/libgo/go/hash/crc32/crc32.go b/libgo/go/hash/crc32/crc32.go
new file mode 100644
index 000000000..2ab0c5491
--- /dev/null
+++ b/libgo/go/hash/crc32/crc32.go
@@ -0,0 +1,111 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This package implements the 32-bit cyclic redundancy check, or CRC-32, checksum.
+// See http://en.wikipedia.org/wiki/Cyclic_redundancy_check for information.
+package crc32
+
+import (
+	"hash"
+	"os"
+)
+
+// The size of a CRC-32 checksum in bytes.
+const Size = 4
+
+// Predefined polynomials.
+const (
+	// Far and away the most common CRC-32 polynomial.
+	// Used by ethernet (IEEE 802.3), v.42, fddi, gzip, zip, png, mpeg-2, ...
+	IEEE = 0xedb88320
+
+	// Castagnoli's polynomial, used in iSCSI.
+	// Has better error detection characteristics than IEEE.
+	// http://dx.doi.org/10.1109/26.231911
+	Castagnoli = 0x82f63b78
+
+	// Koopman's polynomial.
+	// Also has better error detection characteristics than IEEE.
+	// http://dx.doi.org/10.1109/DSN.2002.1028931
+	Koopman = 0xeb31d82e
+)
+
+// Table is a 256-word table representing the polynomial for efficient processing.
+type Table [256]uint32
+
+// MakeTable returns the Table constructed from the specified polynomial.
+func MakeTable(poly uint32) *Table {
+	t := new(Table)
+	for i := 0; i < 256; i++ {
+		crc := uint32(i)
+		for j := 0; j < 8; j++ {
+			if crc&1 == 1 {
+				crc = (crc >> 1) ^ poly
+			} else {
+				crc >>= 1
+			}
+		}
+		t[i] = crc
+	}
+	return t
+}
+
+// IEEETable is the table for the IEEE polynomial.
+var IEEETable = MakeTable(IEEE)
+
+// digest represents the partial evaluation of a checksum.
+type digest struct {
+	crc uint32
+	tab *Table
+}
+
+// New creates a new hash.Hash32 computing the CRC-32 checksum
+// using the polynomial represented by the Table.
+func New(tab *Table) hash.Hash32 { return &digest{0, tab} }
+
+// NewIEEE creates a new hash.Hash32 computing the CRC-32 checksum
+// using the IEEE polynomial.
+func NewIEEE() hash.Hash32 { return New(IEEETable) }
+
+func (d *digest) Size() int { return Size }
+
+func (d *digest) Reset() { d.crc = 0 }
+
+func update(crc uint32, tab *Table, p []byte) uint32 {
+	crc = ^crc
+	for _, v := range p {
+		crc = tab[byte(crc)^v] ^ (crc >> 8)
+	}
+	return ^crc
+}
+
+// Update returns the result of adding the bytes in p to the crc.
+func Update(crc uint32, tab *Table, p []byte) uint32 {
+	return update(crc, tab, p)
+}
+
+func (d *digest) Write(p []byte) (n int, err os.Error) {
+	d.crc = update(d.crc, d.tab, p)
+	return len(p), nil
+}
+
+func (d *digest) Sum32() uint32 { return d.crc }
+
+func (d *digest) Sum() []byte {
+	p := make([]byte, 4)
+	s := d.Sum32()
+	p[0] = byte(s >> 24)
+	p[1] = byte(s >> 16)
+	p[2] = byte(s >> 8)
+	p[3] = byte(s)
+	return p
+}
+
+// Checksum returns the CRC-32 checksum of data
+// using the polynomial represented by the Table.
+func Checksum(data []byte, tab *Table) uint32 { return update(0, tab, data) }
+
+// ChecksumIEEE returns the CRC-32 checksum of data
+// using the IEEE polynomial.
+func ChecksumIEEE(data []byte) uint32 { return update(0, IEEETable, data) }
diff --git a/libgo/go/hash/crc32/crc32_test.go b/libgo/go/hash/crc32/crc32_test.go
new file mode 100644
index 000000000..cf5743c99
--- /dev/null
+++ b/libgo/go/hash/crc32/crc32_test.go
@@ -0,0 +1,76 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package crc32
+
+import (
+	"io"
+	"testing"
+)
+
+type test struct {
+	out uint32
+	in  string
+}
+
+var golden = []test{
+	{0x0, ""},
+	{0xe8b7be43, "a"},
+	{0x9e83486d, "ab"},
+	{0x352441c2, "abc"},
+	{0xed82cd11, "abcd"},
+	{0x8587d865, "abcde"},
+	{0x4b8e39ef, "abcdef"},
+	{0x312a6aa6, "abcdefg"},
+	{0xaeef2a50, "abcdefgh"},
+	{0x8da988af, "abcdefghi"},
+	{0x3981703a, "abcdefghij"},
+	{0x6b9cdfe7, "Discard medicine more than two years old."},
+	{0xc90ef73f, "He who has a shady past knows that nice guys finish last."},
+	{0xb902341f, "I wouldn't marry him with a ten foot pole."},
+	{0x42080e8, "Free! Free!/A trip/to Mars/for 900/empty jars/Burma Shave"},
+	{0x154c6d11, "The days of the digital watch are numbered.  -Tom Stoppard"},
+	{0x4c418325, "Nepal premier won't resign."},
+	{0x33955150, "For every action there is an equal and opposite government program."},
+	{0x26216a4b, "His money is twice tainted: 'taint yours and 'taint mine."},
+	{0x1abbe45e, "There is no reason for any individual to have a computer in their home. -Ken Olsen, 1977"},
+	{0xc89a94f7, "It's a tiny change to the code and not completely disgusting. - Bob Manchek"},
+	{0xab3abe14, "size:  a.out:  bad magic"},
+	{0xbab102b6, "The major problem is with sendmail.  -Mark Horton"},
+	{0x999149d7, "Give me a rock, paper and scissors and I will move the world.  CCFestoon"},
+	{0x6d52a33c, "If the enemy is within range, then so are you."},
+	{0x90631e8d, "It's well we cannot hear the screams/That we create in others' dreams."},
+	{0x78309130, "You remind me of a TV show, but that's all right: I watch it anyway."},
+	{0x7d0a377f, "C is as portable as Stonehedge!!"},
+	{0x8c79fd79, "Even if I could be Shakespeare, I think I should still choose to be Faraday. - A. Huxley"},
+	{0xa20b7167, "The fugacity of a constituent in a mixture of gases at a given temperature is proportional to its mole fraction.  Lewis-Randall Rule"},
+	{0x8e0bb443, "How can you write a big system without C++?  -Paul Glick"},
+}
+
+func TestGolden(t *testing.T) {
+	for i := 0; i < len(golden); i++ {
+		g := golden[i]
+		c := NewIEEE()
+		io.WriteString(c, g.in)
+		s := c.Sum32()
+		if s != g.out {
+			t.Errorf("crc32(%s) = 0x%x want 0x%x", g.in, s, g.out)
+			t.FailNow()
+		}
+	}
+}
+
+func BenchmarkCrc32KB(b *testing.B) {
+	b.StopTimer()
+	data := make([]uint8, 1024)
+	for i := 0; i < 1024; i++ {
+		data[i] = uint8(i)
+	}
+	c := NewIEEE()
+	b.StartTimer()
+
+	for i := 0; i < b.N; i++ {
+		c.Write(data)
+	}
+}
diff --git a/libgo/go/hash/crc64/crc64.go b/libgo/go/hash/crc64/crc64.go
new file mode 100644
index 000000000..844386564
--- /dev/null
+++ b/libgo/go/hash/crc64/crc64.go
@@ -0,0 +1,96 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This package implements the 64-bit cyclic redundancy check, or CRC-64, checksum.
+// See http://en.wikipedia.org/wiki/Cyclic_redundancy_check for information.
+package crc64
+
+import (
+	"hash"
+	"os"
+)
+
+// The size of a CRC-64 checksum in bytes.
+const Size = 8
+
+// Predefined polynomials.
+const (
+	// The ISO polynomial, defined in ISO 3309 and used in HDLC.
+	ISO = 0xD800000000000000
+
+	// The ECMA polynomial, defined in ECMA 182.
+	ECMA = 0xC96C5795D7870F42
+)
+
+// Table is a 256-word table representing the polynomial for efficient processing.
+type Table [256]uint64
+
+// MakeTable returns the Table constructed from the specified polynomial.
+func MakeTable(poly uint64) *Table {
+	t := new(Table)
+	for i := 0; i < 256; i++ {
+		crc := uint64(i)
+		for j := 0; j < 8; j++ {
+			if crc&1 == 1 {
+				crc = (crc >> 1) ^ poly
+			} else {
+				crc >>= 1
+			}
+		}
+		t[i] = crc
+	}
+	return t
+}
+
+// digest represents the partial evaluation of a checksum.
+type digest struct {
+	crc uint64
+	tab *Table
+}
+
+// New creates a new hash.Hash64 computing the CRC-64 checksum
+// using the polynomial represented by the Table.
+func New(tab *Table) hash.Hash64 { return &digest{0, tab} }
+
+func (d *digest) Size() int { return Size }
+
+func (d *digest) Reset() { d.crc = 0 }
+
+func update(crc uint64, tab *Table, p []byte) uint64 {
+	crc = ^crc
+	for _, v := range p {
+		crc = tab[byte(crc)^v] ^ (crc >> 8)
+	}
+	return ^crc
+}
+
+// Update returns the result of adding the bytes in p to the crc.
+func Update(crc uint64, tab *Table, p []byte) uint64 {
+	return update(crc, tab, p)
+}
+
+func (d *digest) Write(p []byte) (n int, err os.Error) {
+	d.crc = update(d.crc, d.tab, p)
+	return len(p), nil
+}
+
+func (d *digest) Sum64() uint64 { return d.crc }
+
+func (d *digest) Sum() []byte {
+	p := make([]byte, 8)
+	s := d.Sum64()
+	p[0] = byte(s >> 56)
+	p[1] = byte(s >> 48)
+	p[2] = byte(s >> 40)
+	p[3] = byte(s >> 32)
+	p[4] = byte(s >> 24)
+	p[5] = byte(s >> 16)
+	p[6] = byte(s >> 8)
+	p[7] = byte(s)
+	return p
+}
+
+// Checksum returns the CRC-64 checksum of data
+// using the polynomial represented by the Table.
+func Checksum(data []byte, tab *Table) uint64 { return update(0, tab, data) }
diff --git a/libgo/go/hash/crc64/crc64_test.go b/libgo/go/hash/crc64/crc64_test.go
new file mode 100644
index 000000000..e932524e0
--- /dev/null
+++ b/libgo/go/hash/crc64/crc64_test.go
@@ -0,0 +1,78 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package crc64
+
+import (
+	"io"
+	"testing"
+)
+
+type test struct {
+	out uint64
+	in  string
+}
+
+var golden = []test{
+	{0x0, ""},
+	{0x3420000000000000, "a"},
+	{0x36c4200000000000, "ab"},
+	{0x3776c42000000000, "abc"},
+	{0x336776c420000000, "abcd"},
+	{0x32d36776c4200000, "abcde"},
+	{0x3002d36776c42000, "abcdef"},
+	{0x31b002d36776c420, "abcdefg"},
+	{0xe21b002d36776c4, "abcdefgh"},
+	{0x8b6e21b002d36776, "abcdefghi"},
+	{0x7f5b6e21b002d367, "abcdefghij"},
+	{0x8ec0e7c835bf9cdf, "Discard medicine more than two years old."},
+	{0xc7db1759e2be5ab4, "He who has a shady past knows that nice guys finish last."},
+	{0xfbf9d9603a6fa020, "I wouldn't marry him with a ten foot pole."},
+	{0xeafc4211a6daa0ef, "Free! Free!/A trip/to Mars/for 900/empty jars/Burma Shave"},
+	{0x3e05b21c7a4dc4da, "The days of the digital watch are numbered.  -Tom Stoppard"},
+	{0x5255866ad6ef28a6, "Nepal premier won't resign."},
+	{0x8a79895be1e9c361, "For every action there is an equal and opposite government program."},
+	{0x8878963a649d4916, "His money is twice tainted: 'taint yours and 'taint mine."},
+	{0xa7b9d53ea87eb82f, "There is no reason for any individual to have a computer in their home. -Ken Olsen, 1977"},
+	{0xdb6805c0966a2f9c, "It's a tiny change to the code and not completely disgusting. - Bob Manchek"},
+	{0xf3553c65dacdadd2, "size:  a.out:  bad magic"},
+	{0x9d5e034087a676b9, "The major problem is with sendmail.  -Mark Horton"},
+	{0xa6db2d7f8da96417, "Give me a rock, paper and scissors and I will move the world.  CCFestoon"},
+	{0x325e00cd2fe819f9, "If the enemy is within range, then so are you."},
+	{0x88c6600ce58ae4c6, "It's well we cannot hear the screams/That we create in others' dreams."},
+	{0x28c4a3f3b769e078, "You remind me of a TV show, but that's all right: I watch it anyway."},
+	{0xa698a34c9d9f1dca, "C is as portable as Stonehedge!!"},
+	{0xf6c1e2a8c26c5cfc, "Even if I could be Shakespeare, I think I should still choose to be Faraday. - A. Huxley"},
+	{0xd402559dfe9b70c, "The fugacity of a constituent in a mixture of gases at a given temperature is proportional to its mole fraction.  Lewis-Randall Rule"},
+	{0xdb6efff26aa94946, "How can you write a big system without C++?  -Paul Glick"},
+}
+
+var tab = MakeTable(ISO)
+
+func TestGolden(t *testing.T) {
+	for i := 0; i < len(golden); i++ {
+		g := golden[i]
+		c := New(tab)
+		io.WriteString(c, g.in)
+		s := c.Sum64()
+		if s != g.out {
+			t.Errorf("crc64(%s) = 0x%x want 0x%x", g.in, s, g.out)
+			t.FailNow()
+		}
+	}
+}
+
+func BenchmarkCrc64KB(b *testing.B) {
+	b.StopTimer()
+	data := make([]uint8, 1024)
+	for i := 0; i < 1024; i++ {
+		data[i] = uint8(i)
+	}
+	c := New(tab)
+	b.StartTimer()
+
+	for i := 0; i < b.N; i++ {
+		c.Write(data)
+	}
+}
diff --git a/libgo/go/hash/hash.go b/libgo/go/hash/hash.go
new file mode 100644
index 000000000..56ac259db
--- /dev/null
+++ b/libgo/go/hash/hash.go
@@ -0,0 +1,36 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package hash
+
+import "io"
+
+// Hash is the common interface implemented by all hash functions.
+type Hash interface {
+	// Write adds more data to the running hash.
+	// It never returns an error.
+	io.Writer
+
+	// Sum returns the current hash, without changing the
+	// underlying hash state.
+	Sum() []byte
+
+	// Reset resets the hash to one with zero bytes written.
+	Reset()
+
+	// Size returns the number of bytes Sum will return.
+	Size() int
+}
+
+// Hash32 is the common interface implemented by all 32-bit hash functions.
+type Hash32 interface {
+	Hash
+	Sum32() uint32
+}
+
+// Hash64 is the common interface implemented by all 64-bit hash functions.
+type Hash64 interface {
+	Hash
+	Sum64() uint64
+}
diff --git a/libgo/go/html/doc.go b/libgo/go/html/doc.go
new file mode 100644
index 000000000..c5338d078
--- /dev/null
+++ b/libgo/go/html/doc.go
@@ -0,0 +1,106 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+/*
+The html package implements an HTML5-compliant tokenizer and parser.
+
+Tokenization is done by creating a Tokenizer for an io.Reader r. It is the
+caller's responsibility to ensure that r provides UTF-8 encoded HTML.
+
+	z := html.NewTokenizer(r)
+
+Given a Tokenizer z, the HTML is tokenized by repeatedly calling z.Next(),
+which parses the next token and returns its type, or an error:
+
+	for {
+		tt := z.Next()
+		if tt == html.ErrorToken {
+			// ...
+			return ...
+		}
+		// Process the current token.
+	}
+
+There are two APIs for retrieving the current token. The high-level API is to
+call Token; the low-level API is to call Text or TagName / TagAttr. Both APIs
+allow optionally calling Raw after Next but before Token, Text, TagName, or
+TagAttr. In EBNF notation, the valid call sequence per token is:
+
+	Next {Raw} [ Token | Text | TagName {TagAttr} ]
+
+Token returns an independent data structure that completely describes a token.
+Entities (such as "<") are unescaped, tag names and attribute keys are
+lower-cased, and attributes are collected into a []Attribute. For example:
+
+	for {
+		if z.Next() == html.ErrorToken {
+			// Returning os.EOF indicates success.
+			return z.Error()
+		}
+		emitToken(z.Token())
+	}
+
+The low-level API performs fewer allocations and copies, but the contents of
+the []byte values returned by Text, TagName and TagAttr may change on the next
+call to Next. For example, to extract an HTML page's anchor text:
+
+	depth := 0
+	for {
+		tt := z.Next()
+		switch tt {
+		case ErrorToken:
+			return z.Error()
+		case TextToken:
+			if depth > 0 {
+				// emitBytes should copy the []byte it receives,
+				// if it doesn't process it immediately.
+				emitBytes(z.Text())
+			}
+		case StartTagToken, EndTagToken:
+			tn, _ := z.TagName()
+			if len(tn) == 1 && tn[0] == 'a' {
+				if tt == StartTag {
+					depth++
+				} else {
+					depth--
+				}
+			}
+		}
+	}
+
+Parsing is done by calling Parse with an io.Reader, which returns the root of
+the parse tree (the document element) as a *Node. It is the caller's
+responsibility to ensure that the Reader provides UTF-8 encoded HTML. For
+example, to process each anchor node in depth-first order:
+
+	doc, err := html.Parse(r)
+	if err != nil {
+		// ...
+	}
+	var f func(*html.Node)
+	f = func(n *html.Node) {
+		if n.Type == html.ElementNode && n.Data == "a" {
+			// Do something with n...
+		}
+		for _, c := range n.Child {
+			f(c)
+		}
+	}
+	f(doc)
+
+The relevant specifications include:
+http://www.whatwg.org/specs/web-apps/current-work/multipage/syntax.html and
+http://www.whatwg.org/specs/web-apps/current-work/multipage/tokenization.html
+*/
+package html
+
+// The tokenization algorithm implemented by this package is not a line-by-line
+// transliteration of the relatively verbose state-machine in the WHATWG
+// specification. A more direct approach is used instead, where the program
+// counter implies the state, such as whether it is tokenizing a tag or a text
+// node. Specification compliance is verified by checking expected and actual
+// outputs over a test suite rather than aiming for algorithmic fidelity.
+
+// TODO(nigeltao): Does a DOM API belong in this package or a separate one?
+// TODO(nigeltao): How does parsing interact with a JavaScript engine?
diff --git a/libgo/go/html/entity.go b/libgo/go/html/entity.go
new file mode 100644
index 000000000..1530290cb
--- /dev/null
+++ b/libgo/go/html/entity.go
@@ -0,0 +1,2250 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package html
+
+// entity is a map from HTML entity names to their values. The semicolon matters:
+// http://www.whatwg.org/specs/web-apps/current-work/multipage/named-character-references.html
+// lists both "amp" and "amp;" as two separate entries.
+//
+// Note that the HTML5 list is larger than the HTML4 list at
+// http://www.w3.org/TR/html4/sgml/entities.html
+var entity = map[string]int{
+	"AElig;":                           '\U000000C6',
+	"AMP;":                             '\U00000026',
+	"Aacute;":                          '\U000000C1',
+	"Abreve;":                          '\U00000102',
+	"Acirc;":                           '\U000000C2',
+	"Acy;":                             '\U00000410',
+	"Afr;":                             '\U0001D504',
+	"Agrave;":                          '\U000000C0',
+	"Alpha;":                           '\U00000391',
+	"Amacr;":                           '\U00000100',
+	"And;":                             '\U00002A53',
+	"Aogon;":                           '\U00000104',
+	"Aopf;":                            '\U0001D538',
+	"ApplyFunction;":                   '\U00002061',
+	"Aring;":                           '\U000000C5',
+	"Ascr;":                            '\U0001D49C',
+	"Assign;":                          '\U00002254',
+	"Atilde;":                          '\U000000C3',
+	"Auml;":                            '\U000000C4',
+	"Backslash;":                       '\U00002216',
+	"Barv;":                            '\U00002AE7',
+	"Barwed;":                          '\U00002306',
+	"Bcy;":                             '\U00000411',
+	"Because;":                         '\U00002235',
+	"Bernoullis;":                      '\U0000212C',
+	"Beta;":                            '\U00000392',
+	"Bfr;":                             '\U0001D505',
+	"Bopf;":                            '\U0001D539',
+	"Breve;":                           '\U000002D8',
+	"Bscr;":                            '\U0000212C',
+	"Bumpeq;":                          '\U0000224E',
+	"CHcy;":                            '\U00000427',
+	"COPY;":                            '\U000000A9',
+	"Cacute;":                          '\U00000106',
+	"Cap;":                             '\U000022D2',
+	"CapitalDifferentialD;":            '\U00002145',
+	"Cayleys;":                         '\U0000212D',
+	"Ccaron;":                          '\U0000010C',
+	"Ccedil;":                          '\U000000C7',
+	"Ccirc;":                           '\U00000108',
+	"Cconint;":                         '\U00002230',
+	"Cdot;":                            '\U0000010A',
+	"Cedilla;":                         '\U000000B8',
+	"CenterDot;":                       '\U000000B7',
+	"Cfr;":                             '\U0000212D',
+	"Chi;":                             '\U000003A7',
+	"CircleDot;":                       '\U00002299',
+	"CircleMinus;":                     '\U00002296',
+	"CirclePlus;":                      '\U00002295',
+	"CircleTimes;":                     '\U00002297',
+	"ClockwiseContourIntegral;":        '\U00002232',
+	"CloseCurlyDoubleQuote;":           '\U0000201D',
+	"CloseCurlyQuote;":                 '\U00002019',
+	"Colon;":                           '\U00002237',
+	"Colone;":                          '\U00002A74',
+	"Congruent;":                       '\U00002261',
+	"Conint;":                          '\U0000222F',
+	"ContourIntegral;":                 '\U0000222E',
+	"Copf;":                            '\U00002102',
+	"Coproduct;":                       '\U00002210',
+	"CounterClockwiseContourIntegral;": '\U00002233',
+	"Cross;":                           '\U00002A2F',
+	"Cscr;":                            '\U0001D49E',
+	"Cup;":                             '\U000022D3',
+	"CupCap;":                          '\U0000224D',
+	"DD;":                              '\U00002145',
+	"DDotrahd;":                        '\U00002911',
+	"DJcy;":                            '\U00000402',
+	"DScy;":                            '\U00000405',
+	"DZcy;":                            '\U0000040F',
+	"Dagger;":                          '\U00002021',
+	"Darr;":                            '\U000021A1',
+	"Dashv;":                           '\U00002AE4',
+	"Dcaron;":                          '\U0000010E',
+	"Dcy;":                             '\U00000414',
+	"Del;":                             '\U00002207',
+	"Delta;":                           '\U00000394',
+	"Dfr;":                             '\U0001D507',
+	"DiacriticalAcute;":                '\U000000B4',
+	"DiacriticalDot;":                  '\U000002D9',
+	"DiacriticalDoubleAcute;":          '\U000002DD',
+	"DiacriticalGrave;":                '\U00000060',
+	"DiacriticalTilde;":                '\U000002DC',
+	"Diamond;":                         '\U000022C4',
+	"DifferentialD;":                   '\U00002146',
+	"Dopf;":                            '\U0001D53B',
+	"Dot;":                             '\U000000A8',
+	"DotDot;":                          '\U000020DC',
+	"DotEqual;":                        '\U00002250',
+	"DoubleContourIntegral;":           '\U0000222F',
+	"DoubleDot;":                       '\U000000A8',
+	"DoubleDownArrow;":                 '\U000021D3',
+	"DoubleLeftArrow;":                 '\U000021D0',
+	"DoubleLeftRightArrow;":            '\U000021D4',
+	"DoubleLeftTee;":                   '\U00002AE4',
+	"DoubleLongLeftArrow;":             '\U000027F8',
+	"DoubleLongLeftRightArrow;":        '\U000027FA',
+	"DoubleLongRightArrow;":            '\U000027F9',
+	"DoubleRightArrow;":                '\U000021D2',
+	"DoubleRightTee;":                  '\U000022A8',
+	"DoubleUpArrow;":                   '\U000021D1',
+	"DoubleUpDownArrow;":               '\U000021D5',
+	"DoubleVerticalBar;":               '\U00002225',
+	"DownArrow;":                       '\U00002193',
+	"DownArrowBar;":                    '\U00002913',
+	"DownArrowUpArrow;":                '\U000021F5',
+	"DownBreve;":                       '\U00000311',
+	"DownLeftRightVector;":             '\U00002950',
+	"DownLeftTeeVector;":               '\U0000295E',
+	"DownLeftVector;":                  '\U000021BD',
+	"DownLeftVectorBar;":               '\U00002956',
+	"DownRightTeeVector;":              '\U0000295F',
+	"DownRightVector;":                 '\U000021C1',
+	"DownRightVectorBar;":              '\U00002957',
+	"DownTee;":                         '\U000022A4',
+	"DownTeeArrow;":                    '\U000021A7',
+	"Downarrow;":                       '\U000021D3',
+	"Dscr;":                            '\U0001D49F',
+	"Dstrok;":                          '\U00000110',
+	"ENG;":                             '\U0000014A',
+	"ETH;":                             '\U000000D0',
+	"Eacute;":                          '\U000000C9',
+	"Ecaron;":                          '\U0000011A',
+	"Ecirc;":                           '\U000000CA',
+	"Ecy;":                             '\U0000042D',
+	"Edot;":                            '\U00000116',
+	"Efr;":                             '\U0001D508',
+	"Egrave;":                          '\U000000C8',
+	"Element;":                         '\U00002208',
+	"Emacr;":                           '\U00000112',
+	"EmptySmallSquare;":                '\U000025FB',
+	"EmptyVerySmallSquare;":            '\U000025AB',
+	"Eogon;":                           '\U00000118',
+	"Eopf;":                            '\U0001D53C',
+	"Epsilon;":                         '\U00000395',
+	"Equal;":                           '\U00002A75',
+	"EqualTilde;":                      '\U00002242',
+	"Equilibrium;":                     '\U000021CC',
+	"Escr;":                            '\U00002130',
+	"Esim;":                            '\U00002A73',
+	"Eta;":                             '\U00000397',
+	"Euml;":                            '\U000000CB',
+	"Exists;":                          '\U00002203',
+	"ExponentialE;":                    '\U00002147',
+	"Fcy;":                             '\U00000424',
+	"Ffr;":                             '\U0001D509',
+	"FilledSmallSquare;":               '\U000025FC',
+	"FilledVerySmallSquare;":           '\U000025AA',
+	"Fopf;":                            '\U0001D53D',
+	"ForAll;":                          '\U00002200',
+	"Fouriertrf;":                      '\U00002131',
+	"Fscr;":                            '\U00002131',
+	"GJcy;":                            '\U00000403',
+	"GT;":                              '\U0000003E',
+	"Gamma;":                           '\U00000393',
+	"Gammad;":                          '\U000003DC',
+	"Gbreve;":                          '\U0000011E',
+	"Gcedil;":                          '\U00000122',
+	"Gcirc;":                           '\U0000011C',
+	"Gcy;":                             '\U00000413',
+	"Gdot;":                            '\U00000120',
+	"Gfr;":                             '\U0001D50A',
+	"Gg;":                              '\U000022D9',
+	"Gopf;":                            '\U0001D53E',
+	"GreaterEqual;":                    '\U00002265',
+	"GreaterEqualLess;":                '\U000022DB',
+	"GreaterFullEqual;":                '\U00002267',
+	"GreaterGreater;":                  '\U00002AA2',
+	"GreaterLess;":                     '\U00002277',
+	"GreaterSlantEqual;":               '\U00002A7E',
+	"GreaterTilde;":                    '\U00002273',
+	"Gscr;":                            '\U0001D4A2',
+	"Gt;":                              '\U0000226B',
+	"HARDcy;":                          '\U0000042A',
+	"Hacek;":                           '\U000002C7',
+	"Hat;":                             '\U0000005E',
+	"Hcirc;":                           '\U00000124',
+	"Hfr;":                             '\U0000210C',
+	"HilbertSpace;":                    '\U0000210B',
+	"Hopf;":                            '\U0000210D',
+	"HorizontalLine;":                  '\U00002500',
+	"Hscr;":                            '\U0000210B',
+	"Hstrok;":                          '\U00000126',
+	"HumpDownHump;":                    '\U0000224E',
+	"HumpEqual;":                       '\U0000224F',
+	"IEcy;":                            '\U00000415',
+	"IJlig;":                           '\U00000132',
+	"IOcy;":                            '\U00000401',
+	"Iacute;":                          '\U000000CD',
+	"Icirc;":                           '\U000000CE',
+	"Icy;":                             '\U00000418',
+	"Idot;":                            '\U00000130',
+	"Ifr;":                             '\U00002111',
+	"Igrave;":                          '\U000000CC',
+	"Im;":                              '\U00002111',
+	"Imacr;":                           '\U0000012A',
+	"ImaginaryI;":                      '\U00002148',
+	"Implies;":                         '\U000021D2',
+	"Int;":                             '\U0000222C',
+	"Integral;":                        '\U0000222B',
+	"Intersection;":                    '\U000022C2',
+	"InvisibleComma;":                  '\U00002063',
+	"InvisibleTimes;":                  '\U00002062',
+	"Iogon;":                           '\U0000012E',
+	"Iopf;":                            '\U0001D540',
+	"Iota;":                            '\U00000399',
+	"Iscr;":                            '\U00002110',
+	"Itilde;":                          '\U00000128',
+	"Iukcy;":                           '\U00000406',
+	"Iuml;":                            '\U000000CF',
+	"Jcirc;":                           '\U00000134',
+	"Jcy;":                             '\U00000419',
+	"Jfr;":                             '\U0001D50D',
+	"Jopf;":                            '\U0001D541',
+	"Jscr;":                            '\U0001D4A5',
+	"Jsercy;":                          '\U00000408',
+	"Jukcy;":                           '\U00000404',
+	"KHcy;":                            '\U00000425',
+	"KJcy;":                            '\U0000040C',
+	"Kappa;":                           '\U0000039A',
+	"Kcedil;":                          '\U00000136',
+	"Kcy;":                             '\U0000041A',
+	"Kfr;":                             '\U0001D50E',
+	"Kopf;":                            '\U0001D542',
+	"Kscr;":                            '\U0001D4A6',
+	"LJcy;":                            '\U00000409',
+	"LT;":                              '\U0000003C',
+	"Lacute;":                          '\U00000139',
+	"Lambda;":                          '\U0000039B',
+	"Lang;":                            '\U000027EA',
+	"Laplacetrf;":                      '\U00002112',
+	"Larr;":                            '\U0000219E',
+	"Lcaron;":                          '\U0000013D',
+	"Lcedil;":                          '\U0000013B',
+	"Lcy;":                             '\U0000041B',
+	"LeftAngleBracket;":                '\U000027E8',
+	"LeftArrow;":                       '\U00002190',
+	"LeftArrowBar;":                    '\U000021E4',
+	"LeftArrowRightArrow;":             '\U000021C6',
+	"LeftCeiling;":                     '\U00002308',
+	"LeftDoubleBracket;":               '\U000027E6',
+	"LeftDownTeeVector;":               '\U00002961',
+	"LeftDownVector;":                  '\U000021C3',
+	"LeftDownVectorBar;":               '\U00002959',
+	"LeftFloor;":                       '\U0000230A',
+	"LeftRightArrow;":                  '\U00002194',
+	"LeftRightVector;":                 '\U0000294E',
+	"LeftTee;":                         '\U000022A3',
+	"LeftTeeArrow;":                    '\U000021A4',
+	"LeftTeeVector;":                   '\U0000295A',
+	"LeftTriangle;":                    '\U000022B2',
+	"LeftTriangleBar;":                 '\U000029CF',
+	"LeftTriangleEqual;":               '\U000022B4',
+	"LeftUpDownVector;":                '\U00002951',
+	"LeftUpTeeVector;":                 '\U00002960',
+	"LeftUpVector;":                    '\U000021BF',
+	"LeftUpVectorBar;":                 '\U00002958',
+	"LeftVector;":                      '\U000021BC',
+	"LeftVectorBar;":                   '\U00002952',
+	"Leftarrow;":                       '\U000021D0',
+	"Leftrightarrow;":                  '\U000021D4',
+	"LessEqualGreater;":                '\U000022DA',
+	"LessFullEqual;":                   '\U00002266',
+	"LessGreater;":                     '\U00002276',
+	"LessLess;":                        '\U00002AA1',
+	"LessSlantEqual;":                  '\U00002A7D',
+	"LessTilde;":                       '\U00002272',
+	"Lfr;":                             '\U0001D50F',
+	"Ll;":                              '\U000022D8',
+	"Lleftarrow;":                      '\U000021DA',
+	"Lmidot;":                          '\U0000013F',
+	"LongLeftArrow;":                   '\U000027F5',
+	"LongLeftRightArrow;":              '\U000027F7',
+	"LongRightArrow;":                  '\U000027F6',
+	"Longleftarrow;":                   '\U000027F8',
+	"Longleftrightarrow;":              '\U000027FA',
+	"Longrightarrow;":                  '\U000027F9',
+	"Lopf;":                            '\U0001D543',
+	"LowerLeftArrow;":                  '\U00002199',
+	"LowerRightArrow;":                 '\U00002198',
+	"Lscr;":                            '\U00002112',
+	"Lsh;":                             '\U000021B0',
+	"Lstrok;":                          '\U00000141',
+	"Lt;":                              '\U0000226A',
+	"Map;":                             '\U00002905',
+	"Mcy;":                             '\U0000041C',
+	"MediumSpace;":                     '\U0000205F',
+	"Mellintrf;":                       '\U00002133',
+	"Mfr;":                             '\U0001D510',
+	"MinusPlus;":                       '\U00002213',
+	"Mopf;":                            '\U0001D544',
+	"Mscr;":                            '\U00002133',
+	"Mu;":                              '\U0000039C',
+	"NJcy;":                            '\U0000040A',
+	"Nacute;":                          '\U00000143',
+	"Ncaron;":                          '\U00000147',
+	"Ncedil;":                          '\U00000145',
+	"Ncy;":                             '\U0000041D',
+	"NegativeMediumSpace;":             '\U0000200B',
+	"NegativeThickSpace;":              '\U0000200B',
+	"NegativeThinSpace;":               '\U0000200B',
+	"NegativeVeryThinSpace;":           '\U0000200B',
+	"NestedGreaterGreater;":            '\U0000226B',
+	"NestedLessLess;":                  '\U0000226A',
+	"NewLine;":                         '\U0000000A',
+	"Nfr;":                             '\U0001D511',
+	"NoBreak;":                         '\U00002060',
+	"NonBreakingSpace;":                '\U000000A0',
+	"Nopf;":                            '\U00002115',
+	"Not;":                             '\U00002AEC',
+	"NotCongruent;":                    '\U00002262',
+	"NotCupCap;":                       '\U0000226D',
+	"NotDoubleVerticalBar;":            '\U00002226',
+	"NotElement;":                      '\U00002209',
+	"NotEqual;":                        '\U00002260',
+	"NotExists;":                       '\U00002204',
+	"NotGreater;":                      '\U0000226F',
+	"NotGreaterEqual;":                 '\U00002271',
+	"NotGreaterLess;":                  '\U00002279',
+	"NotGreaterTilde;":                 '\U00002275',
+	"NotLeftTriangle;":                 '\U000022EA',
+	"NotLeftTriangleEqual;":            '\U000022EC',
+	"NotLess;":                         '\U0000226E',
+	"NotLessEqual;":                    '\U00002270',
+	"NotLessGreater;":                  '\U00002278',
+	"NotLessTilde;":                    '\U00002274',
+	"NotPrecedes;":                     '\U00002280',
+	"NotPrecedesSlantEqual;":           '\U000022E0',
+	"NotReverseElement;":               '\U0000220C',
+	"NotRightTriangle;":                '\U000022EB',
+	"NotRightTriangleEqual;":           '\U000022ED',
+	"NotSquareSubsetEqual;":            '\U000022E2',
+	"NotSquareSupersetEqual;":          '\U000022E3',
+	"NotSubsetEqual;":                  '\U00002288',
+	"NotSucceeds;":                     '\U00002281',
+	"NotSucceedsSlantEqual;":           '\U000022E1',
+	"NotSupersetEqual;":                '\U00002289',
+	"NotTilde;":                        '\U00002241',
+	"NotTildeEqual;":                   '\U00002244',
+	"NotTildeFullEqual;":               '\U00002247',
+	"NotTildeTilde;":                   '\U00002249',
+	"NotVerticalBar;":                  '\U00002224',
+	"Nscr;":                            '\U0001D4A9',
+	"Ntilde;":                          '\U000000D1',
+	"Nu;":                              '\U0000039D',
+	"OElig;":                           '\U00000152',
+	"Oacute;":                          '\U000000D3',
+	"Ocirc;":                           '\U000000D4',
+	"Ocy;":                             '\U0000041E',
+	"Odblac;":                          '\U00000150',
+	"Ofr;":                             '\U0001D512',
+	"Ograve;":                          '\U000000D2',
+	"Omacr;":                           '\U0000014C',
+	"Omega;":                           '\U000003A9',
+	"Omicron;":                         '\U0000039F',
+	"Oopf;":                            '\U0001D546',
+	"OpenCurlyDoubleQuote;":            '\U0000201C',
+	"OpenCurlyQuote;":                  '\U00002018',
+	"Or;":                              '\U00002A54',
+	"Oscr;":                            '\U0001D4AA',
+	"Oslash;":                          '\U000000D8',
+	"Otilde;":                          '\U000000D5',
+	"Otimes;":                          '\U00002A37',
+	"Ouml;":                            '\U000000D6',
+	"OverBar;":                         '\U0000203E',
+	"OverBrace;":                       '\U000023DE',
+	"OverBracket;":                     '\U000023B4',
+	"OverParenthesis;":                 '\U000023DC',
+	"PartialD;":                        '\U00002202',
+	"Pcy;":                             '\U0000041F',
+	"Pfr;":                             '\U0001D513',
+	"Phi;":                             '\U000003A6',
+	"Pi;":                              '\U000003A0',
+	"PlusMinus;":                       '\U000000B1',
+	"Poincareplane;":                   '\U0000210C',
+	"Popf;":                            '\U00002119',
+	"Pr;":                              '\U00002ABB',
+	"Precedes;":                        '\U0000227A',
+	"PrecedesEqual;":                   '\U00002AAF',
+	"PrecedesSlantEqual;":              '\U0000227C',
+	"PrecedesTilde;":                   '\U0000227E',
+	"Prime;":                           '\U00002033',
+	"Product;":                         '\U0000220F',
+	"Proportion;":                      '\U00002237',
+	"Proportional;":                    '\U0000221D',
+	"Pscr;":                            '\U0001D4AB',
+	"Psi;":                             '\U000003A8',
+	"QUOT;":                            '\U00000022',
+	"Qfr;":                             '\U0001D514',
+	"Qopf;":                            '\U0000211A',
+	"Qscr;":                            '\U0001D4AC',
+	"RBarr;":                           '\U00002910',
+	"REG;":                             '\U000000AE',
+	"Racute;":                          '\U00000154',
+	"Rang;":                            '\U000027EB',
+	"Rarr;":                            '\U000021A0',
+	"Rarrtl;":                          '\U00002916',
+	"Rcaron;":                          '\U00000158',
+	"Rcedil;":                          '\U00000156',
+	"Rcy;":                             '\U00000420',
+	"Re;":                              '\U0000211C',
+	"ReverseElement;":                  '\U0000220B',
+	"ReverseEquilibrium;":              '\U000021CB',
+	"ReverseUpEquilibrium;":            '\U0000296F',
+	"Rfr;":                             '\U0000211C',
+	"Rho;":                             '\U000003A1',
+	"RightAngleBracket;":               '\U000027E9',
+	"RightArrow;":                      '\U00002192',
+	"RightArrowBar;":                   '\U000021E5',
+	"RightArrowLeftArrow;":             '\U000021C4',
+	"RightCeiling;":                    '\U00002309',
+	"RightDoubleBracket;":              '\U000027E7',
+	"RightDownTeeVector;":              '\U0000295D',
+	"RightDownVector;":                 '\U000021C2',
+	"RightDownVectorBar;":              '\U00002955',
+	"RightFloor;":                      '\U0000230B',
+	"RightTee;":                        '\U000022A2',
+	"RightTeeArrow;":                   '\U000021A6',
+	"RightTeeVector;":                  '\U0000295B',
+	"RightTriangle;":                   '\U000022B3',
+	"RightTriangleBar;":                '\U000029D0',
+	"RightTriangleEqual;":              '\U000022B5',
+	"RightUpDownVector;":               '\U0000294F',
+	"RightUpTeeVector;":                '\U0000295C',
+	"RightUpVector;":                   '\U000021BE',
+	"RightUpVectorBar;":                '\U00002954',
+	"RightVector;":                     '\U000021C0',
+	"RightVectorBar;":                  '\U00002953',
+	"Rightarrow;":                      '\U000021D2',
+	"Ropf;":                            '\U0000211D',
+	"RoundImplies;":                    '\U00002970',
+	"Rrightarrow;":                     '\U000021DB',
+	"Rscr;":                            '\U0000211B',
+	"Rsh;":                             '\U000021B1',
+	"RuleDelayed;":                     '\U000029F4',
+	"SHCHcy;":                          '\U00000429',
+	"SHcy;":                            '\U00000428',
+	"SOFTcy;":                          '\U0000042C',
+	"Sacute;":                          '\U0000015A',
+	"Sc;":                              '\U00002ABC',
+	"Scaron;":                          '\U00000160',
+	"Scedil;":                          '\U0000015E',
+	"Scirc;":                           '\U0000015C',
+	"Scy;":                             '\U00000421',
+	"Sfr;":                             '\U0001D516',
+	"ShortDownArrow;":                  '\U00002193',
+	"ShortLeftArrow;":                  '\U00002190',
+	"ShortRightArrow;":                 '\U00002192',
+	"ShortUpArrow;":                    '\U00002191',
+	"Sigma;":                           '\U000003A3',
+	"SmallCircle;":                     '\U00002218',
+	"Sopf;":                            '\U0001D54A',
+	"Sqrt;":                            '\U0000221A',
+	"Square;":                          '\U000025A1',
+	"SquareIntersection;":              '\U00002293',
+	"SquareSubset;":                    '\U0000228F',
+	"SquareSubsetEqual;":               '\U00002291',
+	"SquareSuperset;":                  '\U00002290',
+	"SquareSupersetEqual;":             '\U00002292',
+	"SquareUnion;":                     '\U00002294',
+	"Sscr;":                            '\U0001D4AE',
+	"Star;":                            '\U000022C6',
+	"Sub;":                             '\U000022D0',
+	"Subset;":                          '\U000022D0',
+	"SubsetEqual;":                     '\U00002286',
+	"Succeeds;":                        '\U0000227B',
+	"SucceedsEqual;":                   '\U00002AB0',
+	"SucceedsSlantEqual;":              '\U0000227D',
+	"SucceedsTilde;":                   '\U0000227F',
+	"SuchThat;":                        '\U0000220B',
+	"Sum;":                             '\U00002211',
+	"Sup;":                             '\U000022D1',
+	"Superset;":                        '\U00002283',
+	"SupersetEqual;":                   '\U00002287',
+	"Supset;":                          '\U000022D1',
+	"THORN;":                           '\U000000DE',
+	"TRADE;":                           '\U00002122',
+	"TSHcy;":                           '\U0000040B',
+	"TScy;":                            '\U00000426',
+	"Tab;":                             '\U00000009',
+	"Tau;":                             '\U000003A4',
+	"Tcaron;":                          '\U00000164',
+	"Tcedil;":                          '\U00000162',
+	"Tcy;":                             '\U00000422',
+	"Tfr;":                             '\U0001D517',
+	"Therefore;":                       '\U00002234',
+	"Theta;":                           '\U00000398',
+	"ThinSpace;":                       '\U00002009',
+	"Tilde;":                           '\U0000223C',
+	"TildeEqual;":                      '\U00002243',
+	"TildeFullEqual;":                  '\U00002245',
+	"TildeTilde;":                      '\U00002248',
+	"Topf;":                            '\U0001D54B',
+	"TripleDot;":                       '\U000020DB',
+	"Tscr;":                            '\U0001D4AF',
+	"Tstrok;":                          '\U00000166',
+	"Uacute;":                          '\U000000DA',
+	"Uarr;":                            '\U0000219F',
+	"Uarrocir;":                        '\U00002949',
+	"Ubrcy;":                           '\U0000040E',
+	"Ubreve;":                          '\U0000016C',
+	"Ucirc;":                           '\U000000DB',
+	"Ucy;":                             '\U00000423',
+	"Udblac;":                          '\U00000170',
+	"Ufr;":                             '\U0001D518',
+	"Ugrave;":                          '\U000000D9',
+	"Umacr;":                           '\U0000016A',
+	"UnderBar;":                        '\U0000005F',
+	"UnderBrace;":                      '\U000023DF',
+	"UnderBracket;":                    '\U000023B5',
+	"UnderParenthesis;":                '\U000023DD',
+	"Union;":                           '\U000022C3',
+	"UnionPlus;":                       '\U0000228E',
+	"Uogon;":                           '\U00000172',
+	"Uopf;":                            '\U0001D54C',
+	"UpArrow;":                         '\U00002191',
+	"UpArrowBar;":                      '\U00002912',
+	"UpArrowDownArrow;":                '\U000021C5',
+	"UpDownArrow;":                     '\U00002195',
+	"UpEquilibrium;":                   '\U0000296E',
+	"UpTee;":                           '\U000022A5',
+	"UpTeeArrow;":                      '\U000021A5',
+	"Uparrow;":                         '\U000021D1',
+	"Updownarrow;":                     '\U000021D5',
+	"UpperLeftArrow;":                  '\U00002196',
+	"UpperRightArrow;":                 '\U00002197',
+	"Upsi;":                            '\U000003D2',
+	"Upsilon;":                         '\U000003A5',
+	"Uring;":                           '\U0000016E',
+	"Uscr;":                            '\U0001D4B0',
+	"Utilde;":                          '\U00000168',
+	"Uuml;":                            '\U000000DC',
+	"VDash;":                           '\U000022AB',
+	"Vbar;":                            '\U00002AEB',
+	"Vcy;":                             '\U00000412',
+	"Vdash;":                           '\U000022A9',
+	"Vdashl;":                          '\U00002AE6',
+	"Vee;":                             '\U000022C1',
+	"Verbar;":                          '\U00002016',
+	"Vert;":                            '\U00002016',
+	"VerticalBar;":                     '\U00002223',
+	"VerticalLine;":                    '\U0000007C',
+	"VerticalSeparator;":               '\U00002758',
+	"VerticalTilde;":                   '\U00002240',
+	"VeryThinSpace;":                   '\U0000200A',
+	"Vfr;":                             '\U0001D519',
+	"Vopf;":                            '\U0001D54D',
+	"Vscr;":                            '\U0001D4B1',
+	"Vvdash;":                          '\U000022AA',
+	"Wcirc;":                           '\U00000174',
+	"Wedge;":                           '\U000022C0',
+	"Wfr;":                             '\U0001D51A',
+	"Wopf;":                            '\U0001D54E',
+	"Wscr;":                            '\U0001D4B2',
+	"Xfr;":                             '\U0001D51B',
+	"Xi;":                              '\U0000039E',
+	"Xopf;":                            '\U0001D54F',
+	"Xscr;":                            '\U0001D4B3',
+	"YAcy;":                            '\U0000042F',
+	"YIcy;":                            '\U00000407',
+	"YUcy;":                            '\U0000042E',
+	"Yacute;":                          '\U000000DD',
+	"Ycirc;":                           '\U00000176',
+	"Ycy;":                             '\U0000042B',
+	"Yfr;":                             '\U0001D51C',
+	"Yopf;":                            '\U0001D550',
+	"Yscr;":                            '\U0001D4B4',
+	"Yuml;":                            '\U00000178',
+	"ZHcy;":                            '\U00000416',
+	"Zacute;":                          '\U00000179',
+	"Zcaron;":                          '\U0000017D',
+	"Zcy;":                             '\U00000417',
+	"Zdot;":                            '\U0000017B',
+	"ZeroWidthSpace;":                  '\U0000200B',
+	"Zeta;":                            '\U00000396',
+	"Zfr;":                             '\U00002128',
+	"Zopf;":                            '\U00002124',
+	"Zscr;":                            '\U0001D4B5',
+	"aacute;":                          '\U000000E1',
+	"abreve;":                          '\U00000103',
+	"ac;":                              '\U0000223E',
+	"acd;":                             '\U0000223F',
+	"acirc;":                           '\U000000E2',
+	"acute;":                           '\U000000B4',
+	"acy;":                             '\U00000430',
+	"aelig;":                           '\U000000E6',
+	"af;":                              '\U00002061',
+	"afr;":                             '\U0001D51E',
+	"agrave;":                          '\U000000E0',
+	"alefsym;":                         '\U00002135',
+	"aleph;":                           '\U00002135',
+	"alpha;":                           '\U000003B1',
+	"amacr;":                           '\U00000101',
+	"amalg;":                           '\U00002A3F',
+	"amp;":                             '\U00000026',
+	"and;":                             '\U00002227',
+	"andand;":                          '\U00002A55',
+	"andd;":                            '\U00002A5C',
+	"andslope;":                        '\U00002A58',
+	"andv;":                            '\U00002A5A',
+	"ang;":                             '\U00002220',
+	"ange;":                            '\U000029A4',
+	"angle;":                           '\U00002220',
+	"angmsd;":                          '\U00002221',
+	"angmsdaa;":                        '\U000029A8',
+	"angmsdab;":                        '\U000029A9',
+	"angmsdac;":                        '\U000029AA',
+	"angmsdad;":                        '\U000029AB',
+	"angmsdae;":                        '\U000029AC',
+	"angmsdaf;":                        '\U000029AD',
+	"angmsdag;":                        '\U000029AE',
+	"angmsdah;":                        '\U000029AF',
+	"angrt;":                           '\U0000221F',
+	"angrtvb;":                         '\U000022BE',
+	"angrtvbd;":                        '\U0000299D',
+	"angsph;":                          '\U00002222',
+	"angst;":                           '\U000000C5',
+	"angzarr;":                         '\U0000237C',
+	"aogon;":                           '\U00000105',
+	"aopf;":                            '\U0001D552',
+	"ap;":                              '\U00002248',
+	"apE;":                             '\U00002A70',
+	"apacir;":                          '\U00002A6F',
+	"ape;":                             '\U0000224A',
+	"apid;":                            '\U0000224B',
+	"apos;":                            '\U00000027',
+	"approx;":                          '\U00002248',
+	"approxeq;":                        '\U0000224A',
+	"aring;":                           '\U000000E5',
+	"ascr;":                            '\U0001D4B6',
+	"ast;":                             '\U0000002A',
+	"asymp;":                           '\U00002248',
+	"asympeq;":                         '\U0000224D',
+	"atilde;":                          '\U000000E3',
+	"auml;":                            '\U000000E4',
+	"awconint;":                        '\U00002233',
+	"awint;":                           '\U00002A11',
+	"bNot;":                            '\U00002AED',
+	"backcong;":                        '\U0000224C',
+	"backepsilon;":                     '\U000003F6',
+	"backprime;":                       '\U00002035',
+	"backsim;":                         '\U0000223D',
+	"backsimeq;":                       '\U000022CD',
+	"barvee;":                          '\U000022BD',
+	"barwed;":                          '\U00002305',
+	"barwedge;":                        '\U00002305',
+	"bbrk;":                            '\U000023B5',
+	"bbrktbrk;":                        '\U000023B6',
+	"bcong;":                           '\U0000224C',
+	"bcy;":                             '\U00000431',
+	"bdquo;":                           '\U0000201E',
+	"becaus;":                          '\U00002235',
+	"because;":                         '\U00002235',
+	"bemptyv;":                         '\U000029B0',
+	"bepsi;":                           '\U000003F6',
+	"bernou;":                          '\U0000212C',
+	"beta;":                            '\U000003B2',
+	"beth;":                            '\U00002136',
+	"between;":                         '\U0000226C',
+	"bfr;":                             '\U0001D51F',
+	"bigcap;":                          '\U000022C2',
+	"bigcirc;":                         '\U000025EF',
+	"bigcup;":                          '\U000022C3',
+	"bigodot;":                         '\U00002A00',
+	"bigoplus;":                        '\U00002A01',
+	"bigotimes;":                       '\U00002A02',
+	"bigsqcup;":                        '\U00002A06',
+	"bigstar;":                         '\U00002605',
+	"bigtriangledown;":                 '\U000025BD',
+	"bigtriangleup;":                   '\U000025B3',
+	"biguplus;":                        '\U00002A04',
+	"bigvee;":                          '\U000022C1',
+	"bigwedge;":                        '\U000022C0',
+	"bkarow;":                          '\U0000290D',
+	"blacklozenge;":                    '\U000029EB',
+	"blacksquare;":                     '\U000025AA',
+	"blacktriangle;":                   '\U000025B4',
+	"blacktriangledown;":               '\U000025BE',
+	"blacktriangleleft;":               '\U000025C2',
+	"blacktriangleright;":              '\U000025B8',
+	"blank;":                           '\U00002423',
+	"blk12;":                           '\U00002592',
+	"blk14;":                           '\U00002591',
+	"blk34;":                           '\U00002593',
+	"block;":                           '\U00002588',
+	"bnot;":                            '\U00002310',
+	"bopf;":                            '\U0001D553',
+	"bot;":                             '\U000022A5',
+	"bottom;":                          '\U000022A5',
+	"bowtie;":                          '\U000022C8',
+	"boxDL;":                           '\U00002557',
+	"boxDR;":                           '\U00002554',
+	"boxDl;":                           '\U00002556',
+	"boxDr;":                           '\U00002553',
+	"boxH;":                            '\U00002550',
+	"boxHD;":                           '\U00002566',
+	"boxHU;":                           '\U00002569',
+	"boxHd;":                           '\U00002564',
+	"boxHu;":                           '\U00002567',
+	"boxUL;":                           '\U0000255D',
+	"boxUR;":                           '\U0000255A',
+	"boxUl;":                           '\U0000255C',
+	"boxUr;":                           '\U00002559',
+	"boxV;":                            '\U00002551',
+	"boxVH;":                           '\U0000256C',
+	"boxVL;":                           '\U00002563',
+	"boxVR;":                           '\U00002560',
+	"boxVh;":                           '\U0000256B',
+	"boxVl;":                           '\U00002562',
+	"boxVr;":                           '\U0000255F',
+	"boxbox;":                          '\U000029C9',
+	"boxdL;":                           '\U00002555',
+	"boxdR;":                           '\U00002552',
+	"boxdl;":                           '\U00002510',
+	"boxdr;":                           '\U0000250C',
+	"boxh;":                            '\U00002500',
+	"boxhD;":                           '\U00002565',
+	"boxhU;":                           '\U00002568',
+	"boxhd;":                           '\U0000252C',
+	"boxhu;":                           '\U00002534',
+	"boxminus;":                        '\U0000229F',
+	"boxplus;":                         '\U0000229E',
+	"boxtimes;":                        '\U000022A0',
+	"boxuL;":                           '\U0000255B',
+	"boxuR;":                           '\U00002558',
+	"boxul;":                           '\U00002518',
+	"boxur;":                           '\U00002514',
+	"boxv;":                            '\U00002502',
+	"boxvH;":                           '\U0000256A',
+	"boxvL;":                           '\U00002561',
+	"boxvR;":                           '\U0000255E',
+	"boxvh;":                           '\U0000253C',
+	"boxvl;":                           '\U00002524',
+	"boxvr;":                           '\U0000251C',
+	"bprime;":                          '\U00002035',
+	"breve;":                           '\U000002D8',
+	"brvbar;":                          '\U000000A6',
+	"bscr;":                            '\U0001D4B7',
+	"bsemi;":                           '\U0000204F',
+	"bsim;":                            '\U0000223D',
+	"bsime;":                           '\U000022CD',
+	"bsol;":                            '\U0000005C',
+	"bsolb;":                           '\U000029C5',
+	"bsolhsub;":                        '\U000027C8',
+	"bull;":                            '\U00002022',
+	"bullet;":                          '\U00002022',
+	"bump;":                            '\U0000224E',
+	"bumpE;":                           '\U00002AAE',
+	"bumpe;":                           '\U0000224F',
+	"bumpeq;":                          '\U0000224F',
+	"cacute;":                          '\U00000107',
+	"cap;":                             '\U00002229',
+	"capand;":                          '\U00002A44',
+	"capbrcup;":                        '\U00002A49',
+	"capcap;":                          '\U00002A4B',
+	"capcup;":                          '\U00002A47',
+	"capdot;":                          '\U00002A40',
+	"caret;":                           '\U00002041',
+	"caron;":                           '\U000002C7',
+	"ccaps;":                           '\U00002A4D',
+	"ccaron;":                          '\U0000010D',
+	"ccedil;":                          '\U000000E7',
+	"ccirc;":                           '\U00000109',
+	"ccups;":                           '\U00002A4C',
+	"ccupssm;":                         '\U00002A50',
+	"cdot;":                            '\U0000010B',
+	"cedil;":                           '\U000000B8',
+	"cemptyv;":                         '\U000029B2',
+	"cent;":                            '\U000000A2',
+	"centerdot;":                       '\U000000B7',
+	"cfr;":                             '\U0001D520',
+	"chcy;":                            '\U00000447',
+	"check;":                           '\U00002713',
+	"checkmark;":                       '\U00002713',
+	"chi;":                             '\U000003C7',
+	"cir;":                             '\U000025CB',
+	"cirE;":                            '\U000029C3',
+	"circ;":                            '\U000002C6',
+	"circeq;":                          '\U00002257',
+	"circlearrowleft;":                 '\U000021BA',
+	"circlearrowright;":                '\U000021BB',
+	"circledR;":                        '\U000000AE',
+	"circledS;":                        '\U000024C8',
+	"circledast;":                      '\U0000229B',
+	"circledcirc;":                     '\U0000229A',
+	"circleddash;":                     '\U0000229D',
+	"cire;":                            '\U00002257',
+	"cirfnint;":                        '\U00002A10',
+	"cirmid;":                          '\U00002AEF',
+	"cirscir;":                         '\U000029C2',
+	"clubs;":                           '\U00002663',
+	"clubsuit;":                        '\U00002663',
+	"colon;":                           '\U0000003A',
+	"colone;":                          '\U00002254',
+	"coloneq;":                         '\U00002254',
+	"comma;":                           '\U0000002C',
+	"commat;":                          '\U00000040',
+	"comp;":                            '\U00002201',
+	"compfn;":                          '\U00002218',
+	"complement;":                      '\U00002201',
+	"complexes;":                       '\U00002102',
+	"cong;":                            '\U00002245',
+	"congdot;":                         '\U00002A6D',
+	"conint;":                          '\U0000222E',
+	"copf;":                            '\U0001D554',
+	"coprod;":                          '\U00002210',
+	"copy;":                            '\U000000A9',
+	"copysr;":                          '\U00002117',
+	"crarr;":                           '\U000021B5',
+	"cross;":                           '\U00002717',
+	"cscr;":                            '\U0001D4B8',
+	"csub;":                            '\U00002ACF',
+	"csube;":                           '\U00002AD1',
+	"csup;":                            '\U00002AD0',
+	"csupe;":                           '\U00002AD2',
+	"ctdot;":                           '\U000022EF',
+	"cudarrl;":                         '\U00002938',
+	"cudarrr;":                         '\U00002935',
+	"cuepr;":                           '\U000022DE',
+	"cuesc;":                           '\U000022DF',
+	"cularr;":                          '\U000021B6',
+	"cularrp;":                         '\U0000293D',
+	"cup;":                             '\U0000222A',
+	"cupbrcap;":                        '\U00002A48',
+	"cupcap;":                          '\U00002A46',
+	"cupcup;":                          '\U00002A4A',
+	"cupdot;":                          '\U0000228D',
+	"cupor;":                           '\U00002A45',
+	"curarr;":                          '\U000021B7',
+	"curarrm;":                         '\U0000293C',
+	"curlyeqprec;":                     '\U000022DE',
+	"curlyeqsucc;":                     '\U000022DF',
+	"curlyvee;":                        '\U000022CE',
+	"curlywedge;":                      '\U000022CF',
+	"curren;":                          '\U000000A4',
+	"curvearrowleft;":                  '\U000021B6',
+	"curvearrowright;":                 '\U000021B7',
+	"cuvee;":                           '\U000022CE',
+	"cuwed;":                           '\U000022CF',
+	"cwconint;":                        '\U00002232',
+	"cwint;":                           '\U00002231',
+	"cylcty;":                          '\U0000232D',
+	"dArr;":                            '\U000021D3',
+	"dHar;":                            '\U00002965',
+	"dagger;":                          '\U00002020',
+	"daleth;":                          '\U00002138',
+	"darr;":                            '\U00002193',
+	"dash;":                            '\U00002010',
+	"dashv;":                           '\U000022A3',
+	"dbkarow;":                         '\U0000290F',
+	"dblac;":                           '\U000002DD',
+	"dcaron;":                          '\U0000010F',
+	"dcy;":                             '\U00000434',
+	"dd;":                              '\U00002146',
+	"ddagger;":                         '\U00002021',
+	"ddarr;":                           '\U000021CA',
+	"ddotseq;":                         '\U00002A77',
+	"deg;":                             '\U000000B0',
+	"delta;":                           '\U000003B4',
+	"demptyv;":                         '\U000029B1',
+	"dfisht;":                          '\U0000297F',
+	"dfr;":                             '\U0001D521',
+	"dharl;":                           '\U000021C3',
+	"dharr;":                           '\U000021C2',
+	"diam;":                            '\U000022C4',
+	"diamond;":                         '\U000022C4',
+	"diamondsuit;":                     '\U00002666',
+	"diams;":                           '\U00002666',
+	"die;":                             '\U000000A8',
+	"digamma;":                         '\U000003DD',
+	"disin;":                           '\U000022F2',
+	"div;":                             '\U000000F7',
+	"divide;":                          '\U000000F7',
+	"divideontimes;":                   '\U000022C7',
+	"divonx;":                          '\U000022C7',
+	"djcy;":                            '\U00000452',
+	"dlcorn;":                          '\U0000231E',
+	"dlcrop;":                          '\U0000230D',
+	"dollar;":                          '\U00000024',
+	"dopf;":                            '\U0001D555',
+	"dot;":                             '\U000002D9',
+	"doteq;":                           '\U00002250',
+	"doteqdot;":                        '\U00002251',
+	"dotminus;":                        '\U00002238',
+	"dotplus;":                         '\U00002214',
+	"dotsquare;":                       '\U000022A1',
+	"doublebarwedge;":                  '\U00002306',
+	"downarrow;":                       '\U00002193',
+	"downdownarrows;":                  '\U000021CA',
+	"downharpoonleft;":                 '\U000021C3',
+	"downharpoonright;":                '\U000021C2',
+	"drbkarow;":                        '\U00002910',
+	"drcorn;":                          '\U0000231F',
+	"drcrop;":                          '\U0000230C',
+	"dscr;":                            '\U0001D4B9',
+	"dscy;":                            '\U00000455',
+	"dsol;":                            '\U000029F6',
+	"dstrok;":                          '\U00000111',
+	"dtdot;":                           '\U000022F1',
+	"dtri;":                            '\U000025BF',
+	"dtrif;":                           '\U000025BE',
+	"duarr;":                           '\U000021F5',
+	"duhar;":                           '\U0000296F',
+	"dwangle;":                         '\U000029A6',
+	"dzcy;":                            '\U0000045F',
+	"dzigrarr;":                        '\U000027FF',
+	"eDDot;":                           '\U00002A77',
+	"eDot;":                            '\U00002251',
+	"eacute;":                          '\U000000E9',
+	"easter;":                          '\U00002A6E',
+	"ecaron;":                          '\U0000011B',
+	"ecir;":                            '\U00002256',
+	"ecirc;":                           '\U000000EA',
+	"ecolon;":                          '\U00002255',
+	"ecy;":                             '\U0000044D',
+	"edot;":                            '\U00000117',
+	"ee;":                              '\U00002147',
+	"efDot;":                           '\U00002252',
+	"efr;":                             '\U0001D522',
+	"eg;":                              '\U00002A9A',
+	"egrave;":                          '\U000000E8',
+	"egs;":                             '\U00002A96',
+	"egsdot;":                          '\U00002A98',
+	"el;":                              '\U00002A99',
+	"elinters;":                        '\U000023E7',
+	"ell;":                             '\U00002113',
+	"els;":                             '\U00002A95',
+	"elsdot;":                          '\U00002A97',
+	"emacr;":                           '\U00000113',
+	"empty;":                           '\U00002205',
+	"emptyset;":                        '\U00002205',
+	"emptyv;":                          '\U00002205',
+	"emsp;":                            '\U00002003',
+	"emsp13;":                          '\U00002004',
+	"emsp14;":                          '\U00002005',
+	"eng;":                             '\U0000014B',
+	"ensp;":                            '\U00002002',
+	"eogon;":                           '\U00000119',
+	"eopf;":                            '\U0001D556',
+	"epar;":                            '\U000022D5',
+	"eparsl;":                          '\U000029E3',
+	"eplus;":                           '\U00002A71',
+	"epsi;":                            '\U000003B5',
+	"epsilon;":                         '\U000003B5',
+	"epsiv;":                           '\U000003F5',
+	"eqcirc;":                          '\U00002256',
+	"eqcolon;":                         '\U00002255',
+	"eqsim;":                           '\U00002242',
+	"eqslantgtr;":                      '\U00002A96',
+	"eqslantless;":                     '\U00002A95',
+	"equals;":                          '\U0000003D',
+	"equest;":                          '\U0000225F',
+	"equiv;":                           '\U00002261',
+	"equivDD;":                         '\U00002A78',
+	"eqvparsl;":                        '\U000029E5',
+	"erDot;":                           '\U00002253',
+	"erarr;":                           '\U00002971',
+	"escr;":                            '\U0000212F',
+	"esdot;":                           '\U00002250',
+	"esim;":                            '\U00002242',
+	"eta;":                             '\U000003B7',
+	"eth;":                             '\U000000F0',
+	"euml;":                            '\U000000EB',
+	"euro;":                            '\U000020AC',
+	"excl;":                            '\U00000021',
+	"exist;":                           '\U00002203',
+	"expectation;":                     '\U00002130',
+	"exponentiale;":                    '\U00002147',
+	"fallingdotseq;":                   '\U00002252',
+	"fcy;":                             '\U00000444',
+	"female;":                          '\U00002640',
+	"ffilig;":                          '\U0000FB03',
+	"fflig;":                           '\U0000FB00',
+	"ffllig;":                          '\U0000FB04',
+	"ffr;":                             '\U0001D523',
+	"filig;":                           '\U0000FB01',
+	"flat;":                            '\U0000266D',
+	"fllig;":                           '\U0000FB02',
+	"fltns;":                           '\U000025B1',
+	"fnof;":                            '\U00000192',
+	"fopf;":                            '\U0001D557',
+	"forall;":                          '\U00002200',
+	"fork;":                            '\U000022D4',
+	"forkv;":                           '\U00002AD9',
+	"fpartint;":                        '\U00002A0D',
+	"frac12;":                          '\U000000BD',
+	"frac13;":                          '\U00002153',
+	"frac14;":                          '\U000000BC',
+	"frac15;":                          '\U00002155',
+	"frac16;":                          '\U00002159',
+	"frac18;":                          '\U0000215B',
+	"frac23;":                          '\U00002154',
+	"frac25;":                          '\U00002156',
+	"frac34;":                          '\U000000BE',
+	"frac35;":                          '\U00002157',
+	"frac38;":                          '\U0000215C',
+	"frac45;":                          '\U00002158',
+	"frac56;":                          '\U0000215A',
+	"frac58;":                          '\U0000215D',
+	"frac78;":                          '\U0000215E',
+	"frasl;":                           '\U00002044',
+	"frown;":                           '\U00002322',
+	"fscr;":                            '\U0001D4BB',
+	"gE;":                              '\U00002267',
+	"gEl;":                             '\U00002A8C',
+	"gacute;":                          '\U000001F5',
+	"gamma;":                           '\U000003B3',
+	"gammad;":                          '\U000003DD',
+	"gap;":                             '\U00002A86',
+	"gbreve;":                          '\U0000011F',
+	"gcirc;":                           '\U0000011D',
+	"gcy;":                             '\U00000433',
+	"gdot;":                            '\U00000121',
+	"ge;":                              '\U00002265',
+	"gel;":                             '\U000022DB',
+	"geq;":                             '\U00002265',
+	"geqq;":                            '\U00002267',
+	"geqslant;":                        '\U00002A7E',
+	"ges;":                             '\U00002A7E',
+	"gescc;":                           '\U00002AA9',
+	"gesdot;":                          '\U00002A80',
+	"gesdoto;":                         '\U00002A82',
+	"gesdotol;":                        '\U00002A84',
+	"gesles;":                          '\U00002A94',
+	"gfr;":                             '\U0001D524',
+	"gg;":                              '\U0000226B',
+	"ggg;":                             '\U000022D9',
+	"gimel;":                           '\U00002137',
+	"gjcy;":                            '\U00000453',
+	"gl;":                              '\U00002277',
+	"glE;":                             '\U00002A92',
+	"gla;":                             '\U00002AA5',
+	"glj;":                             '\U00002AA4',
+	"gnE;":                             '\U00002269',
+	"gnap;":                            '\U00002A8A',
+	"gnapprox;":                        '\U00002A8A',
+	"gne;":                             '\U00002A88',
+	"gneq;":                            '\U00002A88',
+	"gneqq;":                           '\U00002269',
+	"gnsim;":                           '\U000022E7',
+	"gopf;":                            '\U0001D558',
+	"grave;":                           '\U00000060',
+	"gscr;":                            '\U0000210A',
+	"gsim;":                            '\U00002273',
+	"gsime;":                           '\U00002A8E',
+	"gsiml;":                           '\U00002A90',
+	"gt;":                              '\U0000003E',
+	"gtcc;":                            '\U00002AA7',
+	"gtcir;":                           '\U00002A7A',
+	"gtdot;":                           '\U000022D7',
+	"gtlPar;":                          '\U00002995',
+	"gtquest;":                         '\U00002A7C',
+	"gtrapprox;":                       '\U00002A86',
+	"gtrarr;":                          '\U00002978',
+	"gtrdot;":                          '\U000022D7',
+	"gtreqless;":                       '\U000022DB',
+	"gtreqqless;":                      '\U00002A8C',
+	"gtrless;":                         '\U00002277',
+	"gtrsim;":                          '\U00002273',
+	"hArr;":                            '\U000021D4',
+	"hairsp;":                          '\U0000200A',
+	"half;":                            '\U000000BD',
+	"hamilt;":                          '\U0000210B',
+	"hardcy;":                          '\U0000044A',
+	"harr;":                            '\U00002194',
+	"harrcir;":                         '\U00002948',
+	"harrw;":                           '\U000021AD',
+	"hbar;":                            '\U0000210F',
+	"hcirc;":                           '\U00000125',
+	"hearts;":                          '\U00002665',
+	"heartsuit;":                       '\U00002665',
+	"hellip;":                          '\U00002026',
+	"hercon;":                          '\U000022B9',
+	"hfr;":                             '\U0001D525',
+	"hksearow;":                        '\U00002925',
+	"hkswarow;":                        '\U00002926',
+	"hoarr;":                           '\U000021FF',
+	"homtht;":                          '\U0000223B',
+	"hookleftarrow;":                   '\U000021A9',
+	"hookrightarrow;":                  '\U000021AA',
+	"hopf;":                            '\U0001D559',
+	"horbar;":                          '\U00002015',
+	"hscr;":                            '\U0001D4BD',
+	"hslash;":                          '\U0000210F',
+	"hstrok;":                          '\U00000127',
+	"hybull;":                          '\U00002043',
+	"hyphen;":                          '\U00002010',
+	"iacute;":                          '\U000000ED',
+	"ic;":                              '\U00002063',
+	"icirc;":                           '\U000000EE',
+	"icy;":                             '\U00000438',
+	"iecy;":                            '\U00000435',
+	"iexcl;":                           '\U000000A1',
+	"iff;":                             '\U000021D4',
+	"ifr;":                             '\U0001D526',
+	"igrave;":                          '\U000000EC',
+	"ii;":                              '\U00002148',
+	"iiiint;":                          '\U00002A0C',
+	"iiint;":                           '\U0000222D',
+	"iinfin;":                          '\U000029DC',
+	"iiota;":                           '\U00002129',
+	"ijlig;":                           '\U00000133',
+	"imacr;":                           '\U0000012B',
+	"image;":                           '\U00002111',
+	"imagline;":                        '\U00002110',
+	"imagpart;":                        '\U00002111',
+	"imath;":                           '\U00000131',
+	"imof;":                            '\U000022B7',
+	"imped;":                           '\U000001B5',
+	"in;":                              '\U00002208',
+	"incare;":                          '\U00002105',
+	"infin;":                           '\U0000221E',
+	"infintie;":                        '\U000029DD',
+	"inodot;":                          '\U00000131',
+	"int;":                             '\U0000222B',
+	"intcal;":                          '\U000022BA',
+	"integers;":                        '\U00002124',
+	"intercal;":                        '\U000022BA',
+	"intlarhk;":                        '\U00002A17',
+	"intprod;":                         '\U00002A3C',
+	"iocy;":                            '\U00000451',
+	"iogon;":                           '\U0000012F',
+	"iopf;":                            '\U0001D55A',
+	"iota;":                            '\U000003B9',
+	"iprod;":                           '\U00002A3C',
+	"iquest;":                          '\U000000BF',
+	"iscr;":                            '\U0001D4BE',
+	"isin;":                            '\U00002208',
+	"isinE;":                           '\U000022F9',
+	"isindot;":                         '\U000022F5',
+	"isins;":                           '\U000022F4',
+	"isinsv;":                          '\U000022F3',
+	"isinv;":                           '\U00002208',
+	"it;":                              '\U00002062',
+	"itilde;":                          '\U00000129',
+	"iukcy;":                           '\U00000456',
+	"iuml;":                            '\U000000EF',
+	"jcirc;":                           '\U00000135',
+	"jcy;":                             '\U00000439',
+	"jfr;":                             '\U0001D527',
+	"jmath;":                           '\U00000237',
+	"jopf;":                            '\U0001D55B',
+	"jscr;":                            '\U0001D4BF',
+	"jsercy;":                          '\U00000458',
+	"jukcy;":                           '\U00000454',
+	"kappa;":                           '\U000003BA',
+	"kappav;":                          '\U000003F0',
+	"kcedil;":                          '\U00000137',
+	"kcy;":                             '\U0000043A',
+	"kfr;":                             '\U0001D528',
+	"kgreen;":                          '\U00000138',
+	"khcy;":                            '\U00000445',
+	"kjcy;":                            '\U0000045C',
+	"kopf;":                            '\U0001D55C',
+	"kscr;":                            '\U0001D4C0',
+	"lAarr;":                           '\U000021DA',
+	"lArr;":                            '\U000021D0',
+	"lAtail;":                          '\U0000291B',
+	"lBarr;":                           '\U0000290E',
+	"lE;":                              '\U00002266',
+	"lEg;":                             '\U00002A8B',
+	"lHar;":                            '\U00002962',
+	"lacute;":                          '\U0000013A',
+	"laemptyv;":                        '\U000029B4',
+	"lagran;":                          '\U00002112',
+	"lambda;":                          '\U000003BB',
+	"lang;":                            '\U000027E8',
+	"langd;":                           '\U00002991',
+	"langle;":                          '\U000027E8',
+	"lap;":                             '\U00002A85',
+	"laquo;":                           '\U000000AB',
+	"larr;":                            '\U00002190',
+	"larrb;":                           '\U000021E4',
+	"larrbfs;":                         '\U0000291F',
+	"larrfs;":                          '\U0000291D',
+	"larrhk;":                          '\U000021A9',
+	"larrlp;":                          '\U000021AB',
+	"larrpl;":                          '\U00002939',
+	"larrsim;":                         '\U00002973',
+	"larrtl;":                          '\U000021A2',
+	"lat;":                             '\U00002AAB',
+	"latail;":                          '\U00002919',
+	"late;":                            '\U00002AAD',
+	"lbarr;":                           '\U0000290C',
+	"lbbrk;":                           '\U00002772',
+	"lbrace;":                          '\U0000007B',
+	"lbrack;":                          '\U0000005B',
+	"lbrke;":                           '\U0000298B',
+	"lbrksld;":                         '\U0000298F',
+	"lbrkslu;":                         '\U0000298D',
+	"lcaron;":                          '\U0000013E',
+	"lcedil;":                          '\U0000013C',
+	"lceil;":                           '\U00002308',
+	"lcub;":                            '\U0000007B',
+	"lcy;":                             '\U0000043B',
+	"ldca;":                            '\U00002936',
+	"ldquo;":                           '\U0000201C',
+	"ldquor;":                          '\U0000201E',
+	"ldrdhar;":                         '\U00002967',
+	"ldrushar;":                        '\U0000294B',
+	"ldsh;":                            '\U000021B2',
+	"le;":                              '\U00002264',
+	"leftarrow;":                       '\U00002190',
+	"leftarrowtail;":                   '\U000021A2',
+	"leftharpoondown;":                 '\U000021BD',
+	"leftharpoonup;":                   '\U000021BC',
+	"leftleftarrows;":                  '\U000021C7',
+	"leftrightarrow;":                  '\U00002194',
+	"leftrightarrows;":                 '\U000021C6',
+	"leftrightharpoons;":               '\U000021CB',
+	"leftrightsquigarrow;":             '\U000021AD',
+	"leftthreetimes;":                  '\U000022CB',
+	"leg;":                             '\U000022DA',
+	"leq;":                             '\U00002264',
+	"leqq;":                            '\U00002266',
+	"leqslant;":                        '\U00002A7D',
+	"les;":                             '\U00002A7D',
+	"lescc;":                           '\U00002AA8',
+	"lesdot;":                          '\U00002A7F',
+	"lesdoto;":                         '\U00002A81',
+	"lesdotor;":                        '\U00002A83',
+	"lesges;":                          '\U00002A93',
+	"lessapprox;":                      '\U00002A85',
+	"lessdot;":                         '\U000022D6',
+	"lesseqgtr;":                       '\U000022DA',
+	"lesseqqgtr;":                      '\U00002A8B',
+	"lessgtr;":                         '\U00002276',
+	"lesssim;":                         '\U00002272',
+	"lfisht;":                          '\U0000297C',
+	"lfloor;":                          '\U0000230A',
+	"lfr;":                             '\U0001D529',
+	"lg;":                              '\U00002276',
+	"lgE;":                             '\U00002A91',
+	"lhard;":                           '\U000021BD',
+	"lharu;":                           '\U000021BC',
+	"lharul;":                          '\U0000296A',
+	"lhblk;":                           '\U00002584',
+	"ljcy;":                            '\U00000459',
+	"ll;":                              '\U0000226A',
+	"llarr;":                           '\U000021C7',
+	"llcorner;":                        '\U0000231E',
+	"llhard;":                          '\U0000296B',
+	"lltri;":                           '\U000025FA',
+	"lmidot;":                          '\U00000140',
+	"lmoust;":                          '\U000023B0',
+	"lmoustache;":                      '\U000023B0',
+	"lnE;":                             '\U00002268',
+	"lnap;":                            '\U00002A89',
+	"lnapprox;":                        '\U00002A89',
+	"lne;":                             '\U00002A87',
+	"lneq;":                            '\U00002A87',
+	"lneqq;":                           '\U00002268',
+	"lnsim;":                           '\U000022E6',
+	"loang;":                           '\U000027EC',
+	"loarr;":                           '\U000021FD',
+	"lobrk;":                           '\U000027E6',
+	"longleftarrow;":                   '\U000027F5',
+	"longleftrightarrow;":              '\U000027F7',
+	"longmapsto;":                      '\U000027FC',
+	"longrightarrow;":                  '\U000027F6',
+	"looparrowleft;":                   '\U000021AB',
+	"looparrowright;":                  '\U000021AC',
+	"lopar;":                           '\U00002985',
+	"lopf;":                            '\U0001D55D',
+	"loplus;":                          '\U00002A2D',
+	"lotimes;":                         '\U00002A34',
+	"lowast;":                          '\U00002217',
+	"lowbar;":                          '\U0000005F',
+	"loz;":                             '\U000025CA',
+	"lozenge;":                         '\U000025CA',
+	"lozf;":                            '\U000029EB',
+	"lpar;":                            '\U00000028',
+	"lparlt;":                          '\U00002993',
+	"lrarr;":                           '\U000021C6',
+	"lrcorner;":                        '\U0000231F',
+	"lrhar;":                           '\U000021CB',
+	"lrhard;":                          '\U0000296D',
+	"lrm;":                             '\U0000200E',
+	"lrtri;":                           '\U000022BF',
+	"lsaquo;":                          '\U00002039',
+	"lscr;":                            '\U0001D4C1',
+	"lsh;":                             '\U000021B0',
+	"lsim;":                            '\U00002272',
+	"lsime;":                           '\U00002A8D',
+	"lsimg;":                           '\U00002A8F',
+	"lsqb;":                            '\U0000005B',
+	"lsquo;":                           '\U00002018',
+	"lsquor;":                          '\U0000201A',
+	"lstrok;":                          '\U00000142',
+	"lt;":                              '\U0000003C',
+	"ltcc;":                            '\U00002AA6',
+	"ltcir;":                           '\U00002A79',
+	"ltdot;":                           '\U000022D6',
+	"lthree;":                          '\U000022CB',
+	"ltimes;":                          '\U000022C9',
+	"ltlarr;":                          '\U00002976',
+	"ltquest;":                         '\U00002A7B',
+	"ltrPar;":                          '\U00002996',
+	"ltri;":                            '\U000025C3',
+	"ltrie;":                           '\U000022B4',
+	"ltrif;":                           '\U000025C2',
+	"lurdshar;":                        '\U0000294A',
+	"luruhar;":                         '\U00002966',
+	"mDDot;":                           '\U0000223A',
+	"macr;":                            '\U000000AF',
+	"male;":                            '\U00002642',
+	"malt;":                            '\U00002720',
+	"maltese;":                         '\U00002720',
+	"map;":                             '\U000021A6',
+	"mapsto;":                          '\U000021A6',
+	"mapstodown;":                      '\U000021A7',
+	"mapstoleft;":                      '\U000021A4',
+	"mapstoup;":                        '\U000021A5',
+	"marker;":                          '\U000025AE',
+	"mcomma;":                          '\U00002A29',
+	"mcy;":                             '\U0000043C',
+	"mdash;":                           '\U00002014',
+	"measuredangle;":                   '\U00002221',
+	"mfr;":                             '\U0001D52A',
+	"mho;":                             '\U00002127',
+	"micro;":                           '\U000000B5',
+	"mid;":                             '\U00002223',
+	"midast;":                          '\U0000002A',
+	"midcir;":                          '\U00002AF0',
+	"middot;":                          '\U000000B7',
+	"minus;":                           '\U00002212',
+	"minusb;":                          '\U0000229F',
+	"minusd;":                          '\U00002238',
+	"minusdu;":                         '\U00002A2A',
+	"mlcp;":                            '\U00002ADB',
+	"mldr;":                            '\U00002026',
+	"mnplus;":                          '\U00002213',
+	"models;":                          '\U000022A7',
+	"mopf;":                            '\U0001D55E',
+	"mp;":                              '\U00002213',
+	"mscr;":                            '\U0001D4C2',
+	"mstpos;":                          '\U0000223E',
+	"mu;":                              '\U000003BC',
+	"multimap;":                        '\U000022B8',
+	"mumap;":                           '\U000022B8',
+	"nLeftarrow;":                      '\U000021CD',
+	"nLeftrightarrow;":                 '\U000021CE',
+	"nRightarrow;":                     '\U000021CF',
+	"nVDash;":                          '\U000022AF',
+	"nVdash;":                          '\U000022AE',
+	"nabla;":                           '\U00002207',
+	"nacute;":                          '\U00000144',
+	"nap;":                             '\U00002249',
+	"napos;":                           '\U00000149',
+	"napprox;":                         '\U00002249',
+	"natur;":                           '\U0000266E',
+	"natural;":                         '\U0000266E',
+	"naturals;":                        '\U00002115',
+	"nbsp;":                            '\U000000A0',
+	"ncap;":                            '\U00002A43',
+	"ncaron;":                          '\U00000148',
+	"ncedil;":                          '\U00000146',
+	"ncong;":                           '\U00002247',
+	"ncup;":                            '\U00002A42',
+	"ncy;":                             '\U0000043D',
+	"ndash;":                           '\U00002013',
+	"ne;":                              '\U00002260',
+	"neArr;":                           '\U000021D7',
+	"nearhk;":                          '\U00002924',
+	"nearr;":                           '\U00002197',
+	"nearrow;":                         '\U00002197',
+	"nequiv;":                          '\U00002262',
+	"nesear;":                          '\U00002928',
+	"nexist;":                          '\U00002204',
+	"nexists;":                         '\U00002204',
+	"nfr;":                             '\U0001D52B',
+	"nge;":                             '\U00002271',
+	"ngeq;":                            '\U00002271',
+	"ngsim;":                           '\U00002275',
+	"ngt;":                             '\U0000226F',
+	"ngtr;":                            '\U0000226F',
+	"nhArr;":                           '\U000021CE',
+	"nharr;":                           '\U000021AE',
+	"nhpar;":                           '\U00002AF2',
+	"ni;":                              '\U0000220B',
+	"nis;":                             '\U000022FC',
+	"nisd;":                            '\U000022FA',
+	"niv;":                             '\U0000220B',
+	"njcy;":                            '\U0000045A',
+	"nlArr;":                           '\U000021CD',
+	"nlarr;":                           '\U0000219A',
+	"nldr;":                            '\U00002025',
+	"nle;":                             '\U00002270',
+	"nleftarrow;":                      '\U0000219A',
+	"nleftrightarrow;":                 '\U000021AE',
+	"nleq;":                            '\U00002270',
+	"nless;":                           '\U0000226E',
+	"nlsim;":                           '\U00002274',
+	"nlt;":                             '\U0000226E',
+	"nltri;":                           '\U000022EA',
+	"nltrie;":                          '\U000022EC',
+	"nmid;":                            '\U00002224',
+	"nopf;":                            '\U0001D55F',
+	"not;":                             '\U000000AC',
+	"notin;":                           '\U00002209',
+	"notinva;":                         '\U00002209',
+	"notinvb;":                         '\U000022F7',
+	"notinvc;":                         '\U000022F6',
+	"notni;":                           '\U0000220C',
+	"notniva;":                         '\U0000220C',
+	"notnivb;":                         '\U000022FE',
+	"notnivc;":                         '\U000022FD',
+	"npar;":                            '\U00002226',
+	"nparallel;":                       '\U00002226',
+	"npolint;":                         '\U00002A14',
+	"npr;":                             '\U00002280',
+	"nprcue;":                          '\U000022E0',
+	"nprec;":                           '\U00002280',
+	"nrArr;":                           '\U000021CF',
+	"nrarr;":                           '\U0000219B',
+	"nrightarrow;":                     '\U0000219B',
+	"nrtri;":                           '\U000022EB',
+	"nrtrie;":                          '\U000022ED',
+	"nsc;":                             '\U00002281',
+	"nsccue;":                          '\U000022E1',
+	"nscr;":                            '\U0001D4C3',
+	"nshortmid;":                       '\U00002224',
+	"nshortparallel;":                  '\U00002226',
+	"nsim;":                            '\U00002241',
+	"nsime;":                           '\U00002244',
+	"nsimeq;":                          '\U00002244',
+	"nsmid;":                           '\U00002224',
+	"nspar;":                           '\U00002226',
+	"nsqsube;":                         '\U000022E2',
+	"nsqsupe;":                         '\U000022E3',
+	"nsub;":                            '\U00002284',
+	"nsube;":                           '\U00002288',
+	"nsubseteq;":                       '\U00002288',
+	"nsucc;":                           '\U00002281',
+	"nsup;":                            '\U00002285',
+	"nsupe;":                           '\U00002289',
+	"nsupseteq;":                       '\U00002289',
+	"ntgl;":                            '\U00002279',
+	"ntilde;":                          '\U000000F1',
+	"ntlg;":                            '\U00002278',
+	"ntriangleleft;":                   '\U000022EA',
+	"ntrianglelefteq;":                 '\U000022EC',
+	"ntriangleright;":                  '\U000022EB',
+	"ntrianglerighteq;":                '\U000022ED',
+	"nu;":                              '\U000003BD',
+	"num;":                             '\U00000023',
+	"numero;":                          '\U00002116',
+	"numsp;":                           '\U00002007',
+	"nvDash;":                          '\U000022AD',
+	"nvHarr;":                          '\U00002904',
+	"nvdash;":                          '\U000022AC',
+	"nvinfin;":                         '\U000029DE',
+	"nvlArr;":                          '\U00002902',
+	"nvrArr;":                          '\U00002903',
+	"nwArr;":                           '\U000021D6',
+	"nwarhk;":                          '\U00002923',
+	"nwarr;":                           '\U00002196',
+	"nwarrow;":                         '\U00002196',
+	"nwnear;":                          '\U00002927',
+	"oS;":                              '\U000024C8',
+	"oacute;":                          '\U000000F3',
+	"oast;":                            '\U0000229B',
+	"ocir;":                            '\U0000229A',
+	"ocirc;":                           '\U000000F4',
+	"ocy;":                             '\U0000043E',
+	"odash;":                           '\U0000229D',
+	"odblac;":                          '\U00000151',
+	"odiv;":                            '\U00002A38',
+	"odot;":                            '\U00002299',
+	"odsold;":                          '\U000029BC',
+	"oelig;":                           '\U00000153',
+	"ofcir;":                           '\U000029BF',
+	"ofr;":                             '\U0001D52C',
+	"ogon;":                            '\U000002DB',
+	"ograve;":                          '\U000000F2',
+	"ogt;":                             '\U000029C1',
+	"ohbar;":                           '\U000029B5',
+	"ohm;":                             '\U000003A9',
+	"oint;":                            '\U0000222E',
+	"olarr;":                           '\U000021BA',
+	"olcir;":                           '\U000029BE',
+	"olcross;":                         '\U000029BB',
+	"oline;":                           '\U0000203E',
+	"olt;":                             '\U000029C0',
+	"omacr;":                           '\U0000014D',
+	"omega;":                           '\U000003C9',
+	"omicron;":                         '\U000003BF',
+	"omid;":                            '\U000029B6',
+	"ominus;":                          '\U00002296',
+	"oopf;":                            '\U0001D560',
+	"opar;":                            '\U000029B7',
+	"operp;":                           '\U000029B9',
+	"oplus;":                           '\U00002295',
+	"or;":                              '\U00002228',
+	"orarr;":                           '\U000021BB',
+	"ord;":                             '\U00002A5D',
+	"order;":                           '\U00002134',
+	"orderof;":                         '\U00002134',
+	"ordf;":                            '\U000000AA',
+	"ordm;":                            '\U000000BA',
+	"origof;":                          '\U000022B6',
+	"oror;":                            '\U00002A56',
+	"orslope;":                         '\U00002A57',
+	"orv;":                             '\U00002A5B',
+	"oscr;":                            '\U00002134',
+	"oslash;":                          '\U000000F8',
+	"osol;":                            '\U00002298',
+	"otilde;":                          '\U000000F5',
+	"otimes;":                          '\U00002297',
+	"otimesas;":                        '\U00002A36',
+	"ouml;":                            '\U000000F6',
+	"ovbar;":                           '\U0000233D',
+	"par;":                             '\U00002225',
+	"para;":                            '\U000000B6',
+	"parallel;":                        '\U00002225',
+	"parsim;":                          '\U00002AF3',
+	"parsl;":                           '\U00002AFD',
+	"part;":                            '\U00002202',
+	"pcy;":                             '\U0000043F',
+	"percnt;":                          '\U00000025',
+	"period;":                          '\U0000002E',
+	"permil;":                          '\U00002030',
+	"perp;":                            '\U000022A5',
+	"pertenk;":                         '\U00002031',
+	"pfr;":                             '\U0001D52D',
+	"phi;":                             '\U000003C6',
+	"phiv;":                            '\U000003D5',
+	"phmmat;":                          '\U00002133',
+	"phone;":                           '\U0000260E',
+	"pi;":                              '\U000003C0',
+	"pitchfork;":                       '\U000022D4',
+	"piv;":                             '\U000003D6',
+	"planck;":                          '\U0000210F',
+	"planckh;":                         '\U0000210E',
+	"plankv;":                          '\U0000210F',
+	"plus;":                            '\U0000002B',
+	"plusacir;":                        '\U00002A23',
+	"plusb;":                           '\U0000229E',
+	"pluscir;":                         '\U00002A22',
+	"plusdo;":                          '\U00002214',
+	"plusdu;":                          '\U00002A25',
+	"pluse;":                           '\U00002A72',
+	"plusmn;":                          '\U000000B1',
+	"plussim;":                         '\U00002A26',
+	"plustwo;":                         '\U00002A27',
+	"pm;":                              '\U000000B1',
+	"pointint;":                        '\U00002A15',
+	"popf;":                            '\U0001D561',
+	"pound;":                           '\U000000A3',
+	"pr;":                              '\U0000227A',
+	"prE;":                             '\U00002AB3',
+	"prap;":                            '\U00002AB7',
+	"prcue;":                           '\U0000227C',
+	"pre;":                             '\U00002AAF',
+	"prec;":                            '\U0000227A',
+	"precapprox;":                      '\U00002AB7',
+	"preccurlyeq;":                     '\U0000227C',
+	"preceq;":                          '\U00002AAF',
+	"precnapprox;":                     '\U00002AB9',
+	"precneqq;":                        '\U00002AB5',
+	"precnsim;":                        '\U000022E8',
+	"precsim;":                         '\U0000227E',
+	"prime;":                           '\U00002032',
+	"primes;":                          '\U00002119',
+	"prnE;":                            '\U00002AB5',
+	"prnap;":                           '\U00002AB9',
+	"prnsim;":                          '\U000022E8',
+	"prod;":                            '\U0000220F',
+	"profalar;":                        '\U0000232E',
+	"profline;":                        '\U00002312',
+	"profsurf;":                        '\U00002313',
+	"prop;":                            '\U0000221D',
+	"propto;":                          '\U0000221D',
+	"prsim;":                           '\U0000227E',
+	"prurel;":                          '\U000022B0',
+	"pscr;":                            '\U0001D4C5',
+	"psi;":                             '\U000003C8',
+	"puncsp;":                          '\U00002008',
+	"qfr;":                             '\U0001D52E',
+	"qint;":                            '\U00002A0C',
+	"qopf;":                            '\U0001D562',
+	"qprime;":                          '\U00002057',
+	"qscr;":                            '\U0001D4C6',
+	"quaternions;":                     '\U0000210D',
+	"quatint;":                         '\U00002A16',
+	"quest;":                           '\U0000003F',
+	"questeq;":                         '\U0000225F',
+	"quot;":                            '\U00000022',
+	"rAarr;":                           '\U000021DB',
+	"rArr;":                            '\U000021D2',
+	"rAtail;":                          '\U0000291C',
+	"rBarr;":                           '\U0000290F',
+	"rHar;":                            '\U00002964',
+	"racute;":                          '\U00000155',
+	"radic;":                           '\U0000221A',
+	"raemptyv;":                        '\U000029B3',
+	"rang;":                            '\U000027E9',
+	"rangd;":                           '\U00002992',
+	"range;":                           '\U000029A5',
+	"rangle;":                          '\U000027E9',
+	"raquo;":                           '\U000000BB',
+	"rarr;":                            '\U00002192',
+	"rarrap;":                          '\U00002975',
+	"rarrb;":                           '\U000021E5',
+	"rarrbfs;":                         '\U00002920',
+	"rarrc;":                           '\U00002933',
+	"rarrfs;":                          '\U0000291E',
+	"rarrhk;":                          '\U000021AA',
+	"rarrlp;":                          '\U000021AC',
+	"rarrpl;":                          '\U00002945',
+	"rarrsim;":                         '\U00002974',
+	"rarrtl;":                          '\U000021A3',
+	"rarrw;":                           '\U0000219D',
+	"ratail;":                          '\U0000291A',
+	"ratio;":                           '\U00002236',
+	"rationals;":                       '\U0000211A',
+	"rbarr;":                           '\U0000290D',
+	"rbbrk;":                           '\U00002773',
+	"rbrace;":                          '\U0000007D',
+	"rbrack;":                          '\U0000005D',
+	"rbrke;":                           '\U0000298C',
+	"rbrksld;":                         '\U0000298E',
+	"rbrkslu;":                         '\U00002990',
+	"rcaron;":                          '\U00000159',
+	"rcedil;":                          '\U00000157',
+	"rceil;":                           '\U00002309',
+	"rcub;":                            '\U0000007D',
+	"rcy;":                             '\U00000440',
+	"rdca;":                            '\U00002937',
+	"rdldhar;":                         '\U00002969',
+	"rdquo;":                           '\U0000201D',
+	"rdquor;":                          '\U0000201D',
+	"rdsh;":                            '\U000021B3',
+	"real;":                            '\U0000211C',
+	"realine;":                         '\U0000211B',
+	"realpart;":                        '\U0000211C',
+	"reals;":                           '\U0000211D',
+	"rect;":                            '\U000025AD',
+	"reg;":                             '\U000000AE',
+	"rfisht;":                          '\U0000297D',
+	"rfloor;":                          '\U0000230B',
+	"rfr;":                             '\U0001D52F',
+	"rhard;":                           '\U000021C1',
+	"rharu;":                           '\U000021C0',
+	"rharul;":                          '\U0000296C',
+	"rho;":                             '\U000003C1',
+	"rhov;":                            '\U000003F1',
+	"rightarrow;":                      '\U00002192',
+	"rightarrowtail;":                  '\U000021A3',
+	"rightharpoondown;":                '\U000021C1',
+	"rightharpoonup;":                  '\U000021C0',
+	"rightleftarrows;":                 '\U000021C4',
+	"rightleftharpoons;":               '\U000021CC',
+	"rightrightarrows;":                '\U000021C9',
+	"rightsquigarrow;":                 '\U0000219D',
+	"rightthreetimes;":                 '\U000022CC',
+	"ring;":                            '\U000002DA',
+	"risingdotseq;":                    '\U00002253',
+	"rlarr;":                           '\U000021C4',
+	"rlhar;":                           '\U000021CC',
+	"rlm;":                             '\U0000200F',
+	"rmoust;":                          '\U000023B1',
+	"rmoustache;":                      '\U000023B1',
+	"rnmid;":                           '\U00002AEE',
+	"roang;":                           '\U000027ED',
+	"roarr;":                           '\U000021FE',
+	"robrk;":                           '\U000027E7',
+	"ropar;":                           '\U00002986',
+	"ropf;":                            '\U0001D563',
+	"roplus;":                          '\U00002A2E',
+	"rotimes;":                         '\U00002A35',
+	"rpar;":                            '\U00000029',
+	"rpargt;":                          '\U00002994',
+	"rppolint;":                        '\U00002A12',
+	"rrarr;":                           '\U000021C9',
+	"rsaquo;":                          '\U0000203A',
+	"rscr;":                            '\U0001D4C7',
+	"rsh;":                             '\U000021B1',
+	"rsqb;":                            '\U0000005D',
+	"rsquo;":                           '\U00002019',
+	"rsquor;":                          '\U00002019',
+	"rthree;":                          '\U000022CC',
+	"rtimes;":                          '\U000022CA',
+	"rtri;":                            '\U000025B9',
+	"rtrie;":                           '\U000022B5',
+	"rtrif;":                           '\U000025B8',
+	"rtriltri;":                        '\U000029CE',
+	"ruluhar;":                         '\U00002968',
+	"rx;":                              '\U0000211E',
+	"sacute;":                          '\U0000015B',
+	"sbquo;":                           '\U0000201A',
+	"sc;":                              '\U0000227B',
+	"scE;":                             '\U00002AB4',
+	"scap;":                            '\U00002AB8',
+	"scaron;":                          '\U00000161',
+	"sccue;":                           '\U0000227D',
+	"sce;":                             '\U00002AB0',
+	"scedil;":                          '\U0000015F',
+	"scirc;":                           '\U0000015D',
+	"scnE;":                            '\U00002AB6',
+	"scnap;":                           '\U00002ABA',
+	"scnsim;":                          '\U000022E9',
+	"scpolint;":                        '\U00002A13',
+	"scsim;":                           '\U0000227F',
+	"scy;":                             '\U00000441',
+	"sdot;":                            '\U000022C5',
+	"sdotb;":                           '\U000022A1',
+	"sdote;":                           '\U00002A66',
+	"seArr;":                           '\U000021D8',
+	"searhk;":                          '\U00002925',
+	"searr;":                           '\U00002198',
+	"searrow;":                         '\U00002198',
+	"sect;":                            '\U000000A7',
+	"semi;":                            '\U0000003B',
+	"seswar;":                          '\U00002929',
+	"setminus;":                        '\U00002216',
+	"setmn;":                           '\U00002216',
+	"sext;":                            '\U00002736',
+	"sfr;":                             '\U0001D530',
+	"sfrown;":                          '\U00002322',
+	"sharp;":                           '\U0000266F',
+	"shchcy;":                          '\U00000449',
+	"shcy;":                            '\U00000448',
+	"shortmid;":                        '\U00002223',
+	"shortparallel;":                   '\U00002225',
+	"shy;":                             '\U000000AD',
+	"sigma;":                           '\U000003C3',
+	"sigmaf;":                          '\U000003C2',
+	"sigmav;":                          '\U000003C2',
+	"sim;":                             '\U0000223C',
+	"simdot;":                          '\U00002A6A',
+	"sime;":                            '\U00002243',
+	"simeq;":                           '\U00002243',
+	"simg;":                            '\U00002A9E',
+	"simgE;":                           '\U00002AA0',
+	"siml;":                            '\U00002A9D',
+	"simlE;":                           '\U00002A9F',
+	"simne;":                           '\U00002246',
+	"simplus;":                         '\U00002A24',
+	"simrarr;":                         '\U00002972',
+	"slarr;":                           '\U00002190',
+	"smallsetminus;":                   '\U00002216',
+	"smashp;":                          '\U00002A33',
+	"smeparsl;":                        '\U000029E4',
+	"smid;":                            '\U00002223',
+	"smile;":                           '\U00002323',
+	"smt;":                             '\U00002AAA',
+	"smte;":                            '\U00002AAC',
+	"softcy;":                          '\U0000044C',
+	"sol;":                             '\U0000002F',
+	"solb;":                            '\U000029C4',
+	"solbar;":                          '\U0000233F',
+	"sopf;":                            '\U0001D564',
+	"spades;":                          '\U00002660',
+	"spadesuit;":                       '\U00002660',
+	"spar;":                            '\U00002225',
+	"sqcap;":                           '\U00002293',
+	"sqcup;":                           '\U00002294',
+	"sqsub;":                           '\U0000228F',
+	"sqsube;":                          '\U00002291',
+	"sqsubset;":                        '\U0000228F',
+	"sqsubseteq;":                      '\U00002291',
+	"sqsup;":                           '\U00002290',
+	"sqsupe;":                          '\U00002292',
+	"sqsupset;":                        '\U00002290',
+	"sqsupseteq;":                      '\U00002292',
+	"squ;":                             '\U000025A1',
+	"square;":                          '\U000025A1',
+	"squarf;":                          '\U000025AA',
+	"squf;":                            '\U000025AA',
+	"srarr;":                           '\U00002192',
+	"sscr;":                            '\U0001D4C8',
+	"ssetmn;":                          '\U00002216',
+	"ssmile;":                          '\U00002323',
+	"sstarf;":                          '\U000022C6',
+	"star;":                            '\U00002606',
+	"starf;":                           '\U00002605',
+	"straightepsilon;":                 '\U000003F5',
+	"straightphi;":                     '\U000003D5',
+	"strns;":                           '\U000000AF',
+	"sub;":                             '\U00002282',
+	"subE;":                            '\U00002AC5',
+	"subdot;":                          '\U00002ABD',
+	"sube;":                            '\U00002286',
+	"subedot;":                         '\U00002AC3',
+	"submult;":                         '\U00002AC1',
+	"subnE;":                           '\U00002ACB',
+	"subne;":                           '\U0000228A',
+	"subplus;":                         '\U00002ABF',
+	"subrarr;":                         '\U00002979',
+	"subset;":                          '\U00002282',
+	"subseteq;":                        '\U00002286',
+	"subseteqq;":                       '\U00002AC5',
+	"subsetneq;":                       '\U0000228A',
+	"subsetneqq;":                      '\U00002ACB',
+	"subsim;":                          '\U00002AC7',
+	"subsub;":                          '\U00002AD5',
+	"subsup;":                          '\U00002AD3',
+	"succ;":                            '\U0000227B',
+	"succapprox;":                      '\U00002AB8',
+	"succcurlyeq;":                     '\U0000227D',
+	"succeq;":                          '\U00002AB0',
+	"succnapprox;":                     '\U00002ABA',
+	"succneqq;":                        '\U00002AB6',
+	"succnsim;":                        '\U000022E9',
+	"succsim;":                         '\U0000227F',
+	"sum;":                             '\U00002211',
+	"sung;":                            '\U0000266A',
+	"sup;":                             '\U00002283',
+	"sup1;":                            '\U000000B9',
+	"sup2;":                            '\U000000B2',
+	"sup3;":                            '\U000000B3',
+	"supE;":                            '\U00002AC6',
+	"supdot;":                          '\U00002ABE',
+	"supdsub;":                         '\U00002AD8',
+	"supe;":                            '\U00002287',
+	"supedot;":                         '\U00002AC4',
+	"suphsol;":                         '\U000027C9',
+	"suphsub;":                         '\U00002AD7',
+	"suplarr;":                         '\U0000297B',
+	"supmult;":                         '\U00002AC2',
+	"supnE;":                           '\U00002ACC',
+	"supne;":                           '\U0000228B',
+	"supplus;":                         '\U00002AC0',
+	"supset;":                          '\U00002283',
+	"supseteq;":                        '\U00002287',
+	"supseteqq;":                       '\U00002AC6',
+	"supsetneq;":                       '\U0000228B',
+	"supsetneqq;":                      '\U00002ACC',
+	"supsim;":                          '\U00002AC8',
+	"supsub;":                          '\U00002AD4',
+	"supsup;":                          '\U00002AD6',
+	"swArr;":                           '\U000021D9',
+	"swarhk;":                          '\U00002926',
+	"swarr;":                           '\U00002199',
+	"swarrow;":                         '\U00002199',
+	"swnwar;":                          '\U0000292A',
+	"szlig;":                           '\U000000DF',
+	"target;":                          '\U00002316',
+	"tau;":                             '\U000003C4',
+	"tbrk;":                            '\U000023B4',
+	"tcaron;":                          '\U00000165',
+	"tcedil;":                          '\U00000163',
+	"tcy;":                             '\U00000442',
+	"tdot;":                            '\U000020DB',
+	"telrec;":                          '\U00002315',
+	"tfr;":                             '\U0001D531',
+	"there4;":                          '\U00002234',
+	"therefore;":                       '\U00002234',
+	"theta;":                           '\U000003B8',
+	"thetasym;":                        '\U000003D1',
+	"thetav;":                          '\U000003D1',
+	"thickapprox;":                     '\U00002248',
+	"thicksim;":                        '\U0000223C',
+	"thinsp;":                          '\U00002009',
+	"thkap;":                           '\U00002248',
+	"thksim;":                          '\U0000223C',
+	"thorn;":                           '\U000000FE',
+	"tilde;":                           '\U000002DC',
+	"times;":                           '\U000000D7',
+	"timesb;":                          '\U000022A0',
+	"timesbar;":                        '\U00002A31',
+	"timesd;":                          '\U00002A30',
+	"tint;":                            '\U0000222D',
+	"toea;":                            '\U00002928',
+	"top;":                             '\U000022A4',
+	"topbot;":                          '\U00002336',
+	"topcir;":                          '\U00002AF1',
+	"topf;":                            '\U0001D565',
+	"topfork;":                         '\U00002ADA',
+	"tosa;":                            '\U00002929',
+	"tprime;":                          '\U00002034',
+	"trade;":                           '\U00002122',
+	"triangle;":                        '\U000025B5',
+	"triangledown;":                    '\U000025BF',
+	"triangleleft;":                    '\U000025C3',
+	"trianglelefteq;":                  '\U000022B4',
+	"triangleq;":                       '\U0000225C',
+	"triangleright;":                   '\U000025B9',
+	"trianglerighteq;":                 '\U000022B5',
+	"tridot;":                          '\U000025EC',
+	"trie;":                            '\U0000225C',
+	"triminus;":                        '\U00002A3A',
+	"triplus;":                         '\U00002A39',
+	"trisb;":                           '\U000029CD',
+	"tritime;":                         '\U00002A3B',
+	"trpezium;":                        '\U000023E2',
+	"tscr;":                            '\U0001D4C9',
+	"tscy;":                            '\U00000446',
+	"tshcy;":                           '\U0000045B',
+	"tstrok;":                          '\U00000167',
+	"twixt;":                           '\U0000226C',
+	"twoheadleftarrow;":                '\U0000219E',
+	"twoheadrightarrow;":               '\U000021A0',
+	"uArr;":                            '\U000021D1',
+	"uHar;":                            '\U00002963',
+	"uacute;":                          '\U000000FA',
+	"uarr;":                            '\U00002191',
+	"ubrcy;":                           '\U0000045E',
+	"ubreve;":                          '\U0000016D',
+	"ucirc;":                           '\U000000FB',
+	"ucy;":                             '\U00000443',
+	"udarr;":                           '\U000021C5',
+	"udblac;":                          '\U00000171',
+	"udhar;":                           '\U0000296E',
+	"ufisht;":                          '\U0000297E',
+	"ufr;":                             '\U0001D532',
+	"ugrave;":                          '\U000000F9',
+	"uharl;":                           '\U000021BF',
+	"uharr;":                           '\U000021BE',
+	"uhblk;":                           '\U00002580',
+	"ulcorn;":                          '\U0000231C',
+	"ulcorner;":                        '\U0000231C',
+	"ulcrop;":                          '\U0000230F',
+	"ultri;":                           '\U000025F8',
+	"umacr;":                           '\U0000016B',
+	"uml;":                             '\U000000A8',
+	"uogon;":                           '\U00000173',
+	"uopf;":                            '\U0001D566',
+	"uparrow;":                         '\U00002191',
+	"updownarrow;":                     '\U00002195',
+	"upharpoonleft;":                   '\U000021BF',
+	"upharpoonright;":                  '\U000021BE',
+	"uplus;":                           '\U0000228E',
+	"upsi;":                            '\U000003C5',
+	"upsih;":                           '\U000003D2',
+	"upsilon;":                         '\U000003C5',
+	"upuparrows;":                      '\U000021C8',
+	"urcorn;":                          '\U0000231D',
+	"urcorner;":                        '\U0000231D',
+	"urcrop;":                          '\U0000230E',
+	"uring;":                           '\U0000016F',
+	"urtri;":                           '\U000025F9',
+	"uscr;":                            '\U0001D4CA',
+	"utdot;":                           '\U000022F0',
+	"utilde;":                          '\U00000169',
+	"utri;":                            '\U000025B5',
+	"utrif;":                           '\U000025B4',
+	"uuarr;":                           '\U000021C8',
+	"uuml;":                            '\U000000FC',
+	"uwangle;":                         '\U000029A7',
+	"vArr;":                            '\U000021D5',
+	"vBar;":                            '\U00002AE8',
+	"vBarv;":                           '\U00002AE9',
+	"vDash;":                           '\U000022A8',
+	"vangrt;":                          '\U0000299C',
+	"varepsilon;":                      '\U000003F5',
+	"varkappa;":                        '\U000003F0',
+	"varnothing;":                      '\U00002205',
+	"varphi;":                          '\U000003D5',
+	"varpi;":                           '\U000003D6',
+	"varpropto;":                       '\U0000221D',
+	"varr;":                            '\U00002195',
+	"varrho;":                          '\U000003F1',
+	"varsigma;":                        '\U000003C2',
+	"vartheta;":                        '\U000003D1',
+	"vartriangleleft;":                 '\U000022B2',
+	"vartriangleright;":                '\U000022B3',
+	"vcy;":                             '\U00000432',
+	"vdash;":                           '\U000022A2',
+	"vee;":                             '\U00002228',
+	"veebar;":                          '\U000022BB',
+	"veeeq;":                           '\U0000225A',
+	"vellip;":                          '\U000022EE',
+	"verbar;":                          '\U0000007C',
+	"vert;":                            '\U0000007C',
+	"vfr;":                             '\U0001D533',
+	"vltri;":                           '\U000022B2',
+	"vopf;":                            '\U0001D567',
+	"vprop;":                           '\U0000221D',
+	"vrtri;":                           '\U000022B3',
+	"vscr;":                            '\U0001D4CB',
+	"vzigzag;":                         '\U0000299A',
+	"wcirc;":                           '\U00000175',
+	"wedbar;":                          '\U00002A5F',
+	"wedge;":                           '\U00002227',
+	"wedgeq;":                          '\U00002259',
+	"weierp;":                          '\U00002118',
+	"wfr;":                             '\U0001D534',
+	"wopf;":                            '\U0001D568',
+	"wp;":                              '\U00002118',
+	"wr;":                              '\U00002240',
+	"wreath;":                          '\U00002240',
+	"wscr;":                            '\U0001D4CC',
+	"xcap;":                            '\U000022C2',
+	"xcirc;":                           '\U000025EF',
+	"xcup;":                            '\U000022C3',
+	"xdtri;":                           '\U000025BD',
+	"xfr;":                             '\U0001D535',
+	"xhArr;":                           '\U000027FA',
+	"xharr;":                           '\U000027F7',
+	"xi;":                              '\U000003BE',
+	"xlArr;":                           '\U000027F8',
+	"xlarr;":                           '\U000027F5',
+	"xmap;":                            '\U000027FC',
+	"xnis;":                            '\U000022FB',
+	"xodot;":                           '\U00002A00',
+	"xopf;":                            '\U0001D569',
+	"xoplus;":                          '\U00002A01',
+	"xotime;":                          '\U00002A02',
+	"xrArr;":                           '\U000027F9',
+	"xrarr;":                           '\U000027F6',
+	"xscr;":                            '\U0001D4CD',
+	"xsqcup;":                          '\U00002A06',
+	"xuplus;":                          '\U00002A04',
+	"xutri;":                           '\U000025B3',
+	"xvee;":                            '\U000022C1',
+	"xwedge;":                          '\U000022C0',
+	"yacute;":                          '\U000000FD',
+	"yacy;":                            '\U0000044F',
+	"ycirc;":                           '\U00000177',
+	"ycy;":                             '\U0000044B',
+	"yen;":                             '\U000000A5',
+	"yfr;":                             '\U0001D536',
+	"yicy;":                            '\U00000457',
+	"yopf;":                            '\U0001D56A',
+	"yscr;":                            '\U0001D4CE',
+	"yucy;":                            '\U0000044E',
+	"yuml;":                            '\U000000FF',
+	"zacute;":                          '\U0000017A',
+	"zcaron;":                          '\U0000017E',
+	"zcy;":                             '\U00000437',
+	"zdot;":                            '\U0000017C',
+	"zeetrf;":                          '\U00002128',
+	"zeta;":                            '\U000003B6',
+	"zfr;":                             '\U0001D537',
+	"zhcy;":                            '\U00000436',
+	"zigrarr;":                         '\U000021DD',
+	"zopf;":                            '\U0001D56B',
+	"zscr;":                            '\U0001D4CF',
+	"zwj;":                             '\U0000200D',
+	"zwnj;":                            '\U0000200C',
+	"AElig":                            '\U000000C6',
+	"AMP":                              '\U00000026',
+	"Aacute":                           '\U000000C1',
+	"Acirc":                            '\U000000C2',
+	"Agrave":                           '\U000000C0',
+	"Aring":                            '\U000000C5',
+	"Atilde":                           '\U000000C3',
+	"Auml":                             '\U000000C4',
+	"COPY":                             '\U000000A9',
+	"Ccedil":                           '\U000000C7',
+	"ETH":                              '\U000000D0',
+	"Eacute":                           '\U000000C9',
+	"Ecirc":                            '\U000000CA',
+	"Egrave":                           '\U000000C8',
+	"Euml":                             '\U000000CB',
+	"GT":                               '\U0000003E',
+	"Iacute":                           '\U000000CD',
+	"Icirc":                            '\U000000CE',
+	"Igrave":                           '\U000000CC',
+	"Iuml":                             '\U000000CF',
+	"LT":                               '\U0000003C',
+	"Ntilde":                           '\U000000D1',
+	"Oacute":                           '\U000000D3',
+	"Ocirc":                            '\U000000D4',
+	"Ograve":                           '\U000000D2',
+	"Oslash":                           '\U000000D8',
+	"Otilde":                           '\U000000D5',
+	"Ouml":                             '\U000000D6',
+	"QUOT":                             '\U00000022',
+	"REG":                              '\U000000AE',
+	"THORN":                            '\U000000DE',
+	"Uacute":                           '\U000000DA',
+	"Ucirc":                            '\U000000DB',
+	"Ugrave":                           '\U000000D9',
+	"Uuml":                             '\U000000DC',
+	"Yacute":                           '\U000000DD',
+	"aacute":                           '\U000000E1',
+	"acirc":                            '\U000000E2',
+	"acute":                            '\U000000B4',
+	"aelig":                            '\U000000E6',
+	"agrave":                           '\U000000E0',
+	"amp":                              '\U00000026',
+	"aring":                            '\U000000E5',
+	"atilde":                           '\U000000E3',
+	"auml":                             '\U000000E4',
+	"brvbar":                           '\U000000A6',
+	"ccedil":                           '\U000000E7',
+	"cedil":                            '\U000000B8',
+	"cent":                             '\U000000A2',
+	"copy":                             '\U000000A9',
+	"curren":                           '\U000000A4',
+	"deg":                              '\U000000B0',
+	"divide":                           '\U000000F7',
+	"eacute":                           '\U000000E9',
+	"ecirc":                            '\U000000EA',
+	"egrave":                           '\U000000E8',
+	"eth":                              '\U000000F0',
+	"euml":                             '\U000000EB',
+	"frac12":                           '\U000000BD',
+	"frac14":                           '\U000000BC',
+	"frac34":                           '\U000000BE',
+	"gt":                               '\U0000003E',
+	"iacute":                           '\U000000ED',
+	"icirc":                            '\U000000EE',
+	"iexcl":                            '\U000000A1',
+	"igrave":                           '\U000000EC',
+	"iquest":                           '\U000000BF',
+	"iuml":                             '\U000000EF',
+	"laquo":                            '\U000000AB',
+	"lt":                               '\U0000003C',
+	"macr":                             '\U000000AF',
+	"micro":                            '\U000000B5',
+	"middot":                           '\U000000B7',
+	"nbsp":                             '\U000000A0',
+	"not":                              '\U000000AC',
+	"ntilde":                           '\U000000F1',
+	"oacute":                           '\U000000F3',
+	"ocirc":                            '\U000000F4',
+	"ograve":                           '\U000000F2',
+	"ordf":                             '\U000000AA',
+	"ordm":                             '\U000000BA',
+	"oslash":                           '\U000000F8',
+	"otilde":                           '\U000000F5',
+	"ouml":                             '\U000000F6',
+	"para":                             '\U000000B6',
+	"plusmn":                           '\U000000B1',
+	"pound":                            '\U000000A3',
+	"quot":                             '\U00000022',
+	"raquo":                            '\U000000BB',
+	"reg":                              '\U000000AE',
+	"sect":                             '\U000000A7',
+	"shy":                              '\U000000AD',
+	"sup1":                             '\U000000B9',
+	"sup2":                             '\U000000B2',
+	"sup3":                             '\U000000B3',
+	"szlig":                            '\U000000DF',
+	"thorn":                            '\U000000FE',
+	"times":                            '\U000000D7',
+	"uacute":                           '\U000000FA',
+	"ucirc":                            '\U000000FB',
+	"ugrave":                           '\U000000F9',
+	"uml":                              '\U000000A8',
+	"uuml":                             '\U000000FC',
+	"yacute":                           '\U000000FD',
+	"yen":                              '\U000000A5',
+	"yuml":                             '\U000000FF',
+}
+
+// HTML entities that are two unicode codepoints.
+var entity2 = map[string][2]int{
+	// TODO(nigeltao): Handle replacements that are wider than their names.
+	// "nLt;":                     {'\u226A', '\u20D2'},
+	// "nGt;":                     {'\u226B', '\u20D2'},
+	"NotEqualTilde;":           {'\u2242', '\u0338'},
+	"NotGreaterFullEqual;":     {'\u2267', '\u0338'},
+	"NotGreaterGreater;":       {'\u226B', '\u0338'},
+	"NotGreaterSlantEqual;":    {'\u2A7E', '\u0338'},
+	"NotHumpDownHump;":         {'\u224E', '\u0338'},
+	"NotHumpEqual;":            {'\u224F', '\u0338'},
+	"NotLeftTriangleBar;":      {'\u29CF', '\u0338'},
+	"NotLessLess;":             {'\u226A', '\u0338'},
+	"NotLessSlantEqual;":       {'\u2A7D', '\u0338'},
+	"NotNestedGreaterGreater;": {'\u2AA2', '\u0338'},
+	"NotNestedLessLess;":       {'\u2AA1', '\u0338'},
+	"NotPrecedesEqual;":        {'\u2AAF', '\u0338'},
+	"NotRightTriangleBar;":     {'\u29D0', '\u0338'},
+	"NotSquareSubset;":         {'\u228F', '\u0338'},
+	"NotSquareSuperset;":       {'\u2290', '\u0338'},
+	"NotSubset;":               {'\u2282', '\u20D2'},
+	"NotSucceedsEqual;":        {'\u2AB0', '\u0338'},
+	"NotSucceedsTilde;":        {'\u227F', '\u0338'},
+	"NotSuperset;":             {'\u2283', '\u20D2'},
+	"ThickSpace;":              {'\u205F', '\u200A'},
+	"acE;":                     {'\u223E', '\u0333'},
+	"bne;":                     {'\u003D', '\u20E5'},
+	"bnequiv;":                 {'\u2261', '\u20E5'},
+	"caps;":                    {'\u2229', '\uFE00'},
+	"cups;":                    {'\u222A', '\uFE00'},
+	"fjlig;":                   {'\u0066', '\u006A'},
+	"gesl;":                    {'\u22DB', '\uFE00'},
+	"gvertneqq;":               {'\u2269', '\uFE00'},
+	"gvnE;":                    {'\u2269', '\uFE00'},
+	"lates;":                   {'\u2AAD', '\uFE00'},
+	"lesg;":                    {'\u22DA', '\uFE00'},
+	"lvertneqq;":               {'\u2268', '\uFE00'},
+	"lvnE;":                    {'\u2268', '\uFE00'},
+	"nGg;":                     {'\u22D9', '\u0338'},
+	"nGtv;":                    {'\u226B', '\u0338'},
+	"nLl;":                     {'\u22D8', '\u0338'},
+	"nLtv;":                    {'\u226A', '\u0338'},
+	"nang;":                    {'\u2220', '\u20D2'},
+	"napE;":                    {'\u2A70', '\u0338'},
+	"napid;":                   {'\u224B', '\u0338'},
+	"nbump;":                   {'\u224E', '\u0338'},
+	"nbumpe;":                  {'\u224F', '\u0338'},
+	"ncongdot;":                {'\u2A6D', '\u0338'},
+	"nedot;":                   {'\u2250', '\u0338'},
+	"nesim;":                   {'\u2242', '\u0338'},
+	"ngE;":                     {'\u2267', '\u0338'},
+	"ngeqq;":                   {'\u2267', '\u0338'},
+	"ngeqslant;":               {'\u2A7E', '\u0338'},
+	"nges;":                    {'\u2A7E', '\u0338'},
+	"nlE;":                     {'\u2266', '\u0338'},
+	"nleqq;":                   {'\u2266', '\u0338'},
+	"nleqslant;":               {'\u2A7D', '\u0338'},
+	"nles;":                    {'\u2A7D', '\u0338'},
+	"notinE;":                  {'\u22F9', '\u0338'},
+	"notindot;":                {'\u22F5', '\u0338'},
+	"nparsl;":                  {'\u2AFD', '\u20E5'},
+	"npart;":                   {'\u2202', '\u0338'},
+	"npre;":                    {'\u2AAF', '\u0338'},
+	"npreceq;":                 {'\u2AAF', '\u0338'},
+	"nrarrc;":                  {'\u2933', '\u0338'},
+	"nrarrw;":                  {'\u219D', '\u0338'},
+	"nsce;":                    {'\u2AB0', '\u0338'},
+	"nsubE;":                   {'\u2AC5', '\u0338'},
+	"nsubset;":                 {'\u2282', '\u20D2'},
+	"nsubseteqq;":              {'\u2AC5', '\u0338'},
+	"nsucceq;":                 {'\u2AB0', '\u0338'},
+	"nsupE;":                   {'\u2AC6', '\u0338'},
+	"nsupset;":                 {'\u2283', '\u20D2'},
+	"nsupseteqq;":              {'\u2AC6', '\u0338'},
+	"nvap;":                    {'\u224D', '\u20D2'},
+	"nvge;":                    {'\u2265', '\u20D2'},
+	"nvgt;":                    {'\u003E', '\u20D2'},
+	"nvle;":                    {'\u2264', '\u20D2'},
+	"nvlt;":                    {'\u003C', '\u20D2'},
+	"nvltrie;":                 {'\u22B4', '\u20D2'},
+	"nvrtrie;":                 {'\u22B5', '\u20D2'},
+	"nvsim;":                   {'\u223C', '\u20D2'},
+	"race;":                    {'\u223D', '\u0331'},
+	"smtes;":                   {'\u2AAC', '\uFE00'},
+	"sqcaps;":                  {'\u2293', '\uFE00'},
+	"sqcups;":                  {'\u2294', '\uFE00'},
+	"varsubsetneq;":            {'\u228A', '\uFE00'},
+	"varsubsetneqq;":           {'\u2ACB', '\uFE00'},
+	"varsupsetneq;":            {'\u228B', '\uFE00'},
+	"varsupsetneqq;":           {'\u2ACC', '\uFE00'},
+	"vnsub;":                   {'\u2282', '\u20D2'},
+	"vnsup;":                   {'\u2283', '\u20D2'},
+	"vsubnE;":                  {'\u2ACB', '\uFE00'},
+	"vsubne;":                  {'\u228A', '\uFE00'},
+	"vsupnE;":                  {'\u2ACC', '\uFE00'},
+	"vsupne;":                  {'\u228B', '\uFE00'},
+}
diff --git a/libgo/go/html/entity_test.go b/libgo/go/html/entity_test.go
new file mode 100644
index 000000000..a1eb4d4f0
--- /dev/null
+++ b/libgo/go/html/entity_test.go
@@ -0,0 +1,26 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package html
+
+import (
+	"testing"
+	"utf8"
+)
+
+func TestEntityLength(t *testing.T) {
+	// We verify that the length of UTF-8 encoding of each value is <= 1 + len(key).
+	// The +1 comes from the leading "&". This property implies that the length of
+	// unescaped text is <= the length of escaped text.
+	for k, v := range entity {
+		if 1+len(k) < utf8.RuneLen(v) {
+			t.Error("escaped entity &" + k + " is shorter than its UTF-8 encoding " + string(v))
+		}
+	}
+	for k, v := range entity2 {
+		if 1+len(k) < utf8.RuneLen(v[0])+utf8.RuneLen(v[1]) {
+			t.Error("escaped entity &" + k + " is shorter than its UTF-8 encoding " + string(v[0]) + string(v[1]))
+		}
+	}
+}
diff --git a/libgo/go/html/escape.go b/libgo/go/html/escape.go
new file mode 100644
index 000000000..2799f6908
--- /dev/null
+++ b/libgo/go/html/escape.go
@@ -0,0 +1,224 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package html
+
+import (
+	"bytes"
+	"strings"
+	"utf8"
+)
+
+// These replacements permit compatibility with old numeric entities that 
+// assumed Windows-1252 encoding.
+// http://www.whatwg.org/specs/web-apps/current-work/multipage/tokenization.html#consume-a-character-reference
+var replacementTable = [...]int{
+	'\u20AC', // First entry is what 0x80 should be replaced with.
+	'\u0081',
+	'\u201A',
+	'\u0192',
+	'\u201E',
+	'\u2026',
+	'\u2020',
+	'\u2021',
+	'\u02C6',
+	'\u2030',
+	'\u0160',
+	'\u2039',
+	'\u0152',
+	'\u008D',
+	'\u017D',
+	'\u008F',
+	'\u0090',
+	'\u2018',
+	'\u2019',
+	'\u201C',
+	'\u201D',
+	'\u2022',
+	'\u2013',
+	'\u2014',
+	'\u02DC',
+	'\u2122',
+	'\u0161',
+	'\u203A',
+	'\u0153',
+	'\u009D',
+	'\u017E',
+	'\u0178', // Last entry is 0x9F.
+	// 0x00->'\uFFFD' is handled programmatically. 
+	// 0x0D->'\u000D' is a no-op.
+}
+
+// unescapeEntity reads an entity like "<" from b[src:] and writes the
+// corresponding "<" to b[dst:], returning the incremented dst and src cursors.
+// Precondition: b[src] == '&' && dst <= src.
+func unescapeEntity(b []byte, dst, src int) (dst1, src1 int) {
+	// http://www.whatwg.org/specs/web-apps/current-work/multipage/tokenization.html#consume-a-character-reference
+
+	// i starts at 1 because we already know that s[0] == '&'.
+	i, s := 1, b[src:]
+
+	if len(s) <= 1 {
+		b[dst] = b[src]
+		return dst + 1, src + 1
+	}
+
+	if s[i] == '#' {
+		if len(s) <= 3 { // We need to have at least "&#.".
+			b[dst] = b[src]
+			return dst + 1, src + 1
+		}
+		i++
+		c := s[i]
+		hex := false
+		if c == 'x' || c == 'X' {
+			hex = true
+			i++
+		}
+
+		x := 0
+		for i < len(s) {
+			c = s[i]
+			i++
+			if hex {
+				if '0' <= c && c <= '9' {
+					x = 16*x + int(c) - '0'
+					continue
+				} else if 'a' <= c && c <= 'f' {
+					x = 16*x + int(c) - 'a' + 10
+					continue
+				} else if 'A' <= c && c <= 'F' {
+					x = 16*x + int(c) - 'A' + 10
+					continue
+				}
+			} else if '0' <= c && c <= '9' {
+				x = 10*x + int(c) - '0'
+				continue
+			}
+			if c != ';' {
+				i--
+			}
+			break
+		}
+
+		if i <= 3 { // No characters matched.
+			b[dst] = b[src]
+			return dst + 1, src + 1
+		}
+
+		if 0x80 <= x && x <= 0x9F {
+			// Replace characters from Windows-1252 with UTF-8 equivalents.
+			x = replacementTable[x-0x80]
+		} else if x == 0 || (0xD800 <= x && x <= 0xDFFF) || x > 0x10FFFF {
+			// Replace invalid characters with the replacement character.
+			x = '\uFFFD'
+		}
+
+		return dst + utf8.EncodeRune(b[dst:], x), src + i
+	}
+
+	// Consume the maximum number of characters possible, with the
+	// consumed characters matching one of the named references.
+
+	// TODO(nigeltao): unescape("¬it;") should be "¬it;"
+	for i < len(s) {
+		c := s[i]
+		i++
+		// Lower-cased characters are more common in entities, so we check for them first.
+		if 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' {
+			continue
+		}
+		if c != ';' {
+			i--
+		}
+		break
+	}
+
+	entityName := string(s[1:i])
+	if x := entity[entityName]; x != 0 {
+		return dst + utf8.EncodeRune(b[dst:], x), src + i
+	} else if x := entity2[entityName]; x[0] != 0 { // Check if it's a two-character entity.
+		dst1 := dst + utf8.EncodeRune(b[dst:], x[0])
+		return dst1 + utf8.EncodeRune(b[dst1:], x[1]), src + i
+	}
+
+	dst1, src1 = dst+i, src+i
+	copy(b[dst:dst1], b[src:src1])
+	return dst1, src1
+}
+
+// unescape unescapes b's entities in-place, so that "a<b" becomes "a"`
+
+func escape(buf *bytes.Buffer, s string) {
+	i := strings.IndexAny(s, escapedChars)
+	for i != -1 {
+		buf.WriteString(s[0:i])
+		var esc string
+		switch s[i] {
+		case '&':
+			esc = "&"
+		case '\'':
+			esc = "'"
+		case '<':
+			esc = "<"
+		case '>':
+			esc = ">"
+		case '"':
+			esc = """
+		default:
+			panic("unrecognized escape character")
+		}
+		s = s[i+1:]
+		buf.WriteString(esc)
+		i = strings.IndexAny(s, escapedChars)
+	}
+	buf.WriteString(s)
+}
+
+// EscapeString escapes special characters like "<" to become "<". It
+// escapes only five such characters: amp, apos, lt, gt and quot.
+// UnescapeString(EscapeString(s)) == s always holds, but the converse isn't
+// always true.
+func EscapeString(s string) string {
+	if strings.IndexAny(s, escapedChars) == -1 {
+		return s
+	}
+	buf := bytes.NewBuffer(nil)
+	escape(buf, s)
+	return buf.String()
+}
+
+// UnescapeString unescapes entities like "<" to become "<". It unescapes a
+// larger range of entities than EscapeString escapes. For example, "á"
+// unescapes to "á", as does "á" and "&xE1;".
+// UnescapeString(EscapeString(s)) == s always holds, but the converse isn't
+// always true.
+func UnescapeString(s string) string {
+	for _, c := range s {
+		if c == '&' {
+			return string(unescape([]byte(s)))
+		}
+	}
+	return s
+}
diff --git a/libgo/go/html/parse.go b/libgo/go/html/parse.go
new file mode 100644
index 000000000..2ef90a873
--- /dev/null
+++ b/libgo/go/html/parse.go
@@ -0,0 +1,666 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package html
+
+import (
+	"io"
+	"os"
+)
+
+// A NodeType is the type of a Node.
+type NodeType int
+
+const (
+	ErrorNode NodeType = iota
+	TextNode
+	DocumentNode
+	ElementNode
+	CommentNode
+)
+
+// A Node consists of a NodeType and some Data (tag name for element nodes,
+// content for text) and are part of a tree of Nodes. Element nodes may also
+// contain a slice of Attributes. Data is unescaped, so that it looks like
+// "a are re-interpreted as a two-token sequence:
+	// 
 followed by . hasSelfClosingToken is true if we have just read
+	// the synthetic start tag and the next one due is the matching end tag.
+	hasSelfClosingToken bool
+	// doc is the document root element.
+	doc *Node
+	// The stack of open elements (section 10.2.3.2).
+	stack []*Node
+	// Element pointers (section 10.2.3.4).
+	head, form *Node
+	// Other parsing state flags (section 10.2.3.5).
+	scripting, framesetOK bool
+}
+
+// push pushes onto the stack of open elements.
+func (p *parser) push(n *Node) {
+	p.stack = append(p.stack, n)
+}
+
+// top returns the top of the stack of open elements.
+// This is also known as the current node.
+func (p *parser) top() *Node {
+	if n := len(p.stack); n > 0 {
+		return p.stack[n-1]
+	}
+	return p.doc
+}
+
+// pop pops the top of the stack of open elements.
+// It will panic if the stack is empty.
+func (p *parser) pop() *Node {
+	n := len(p.stack)
+	ret := p.stack[n-1]
+	p.stack = p.stack[:n-1]
+	return ret
+}
+
+// stopTags for use in popUntil. These come from section 10.2.3.2.
+var (
+	defaultScopeStopTags  = []string{"applet", "caption", "html", "table", "td", "th", "marquee", "object"}
+	listItemScopeStopTags = []string{"applet", "caption", "html", "table", "td", "th", "marquee", "object", "ol", "ul"}
+	buttonScopeStopTags   = []string{"applet", "caption", "html", "table", "td", "th", "marquee", "object", "button"}
+	tableScopeStopTags    = []string{"html", "table"}
+)
+
+// popUntil pops the stack of open elements at the highest element whose tag
+// is in matchTags, provided there is no higher element in stopTags. It returns
+// whether or not there was such an element. If there was not, popUntil leaves
+// the stack unchanged.
+//
+// For example, if the stack was:
+// ["html", "body", "font", "table", "b", "i", "u"]
+// then popUntil([]string{"html, "table"}, "font") would return false, but
+// popUntil([]string{"html, "table"}, "i") would return true and the resultant
+// stack would be:
+// ["html", "body", "font", "table", "b"]
+//
+// If an element's tag is in both stopTags and matchTags, then the stack will
+// be popped and the function returns true (provided, of course, there was no
+// higher element in the stack that was also in stopTags). For example,
+// popUntil([]string{"html, "table"}, "table") would return true and leave:
+// ["html", "body", "font"]
+func (p *parser) popUntil(stopTags []string, matchTags ...string) bool {
+	for i := len(p.stack) - 1; i >= 0; i-- {
+		tag := p.stack[i].Data
+		for _, t := range matchTags {
+			if t == tag {
+				p.stack = p.stack[:i]
+				return true
+			}
+		}
+		for _, t := range stopTags {
+			if t == tag {
+				return false
+			}
+		}
+	}
+	return false
+}
+
+// addChild adds a child node n to the top element, and pushes n if it is an
+// element node (text nodes are not part of the stack of open elements).
+func (p *parser) addChild(n *Node) {
+	m := p.top()
+	m.Child = append(m.Child, n)
+	if n.Type == ElementNode {
+		p.push(n)
+	}
+}
+
+// addText calls addChild with a text node.
+func (p *parser) addText(text string) {
+	// TODO: merge s with previous text, if the preceding node is a text node.
+	// TODO: distinguish whitespace text from others.
+	p.addChild(&Node{
+		Type: TextNode,
+		Data: text,
+	})
+}
+
+// addElement calls addChild with an element node.
+func (p *parser) addElement(tag string, attr []Attribute) {
+	p.addChild(&Node{
+		Type: ElementNode,
+		Data: tag,
+		Attr: attr,
+	})
+}
+
+// Section 10.2.3.3.
+func (p *parser) addFormattingElement(tag string, attr []Attribute) {
+	p.addElement(tag, attr)
+	// TODO.
+}
+
+// Section 10.2.3.3.
+func (p *parser) reconstructActiveFormattingElements() {
+	// TODO.
+}
+
+// read reads the next token. This is usually from the tokenizer, but it may
+// be the synthesized end tag implied by a self-closing tag.
+func (p *parser) read() os.Error {
+	if p.hasSelfClosingToken {
+		p.hasSelfClosingToken = false
+		p.tok.Type = EndTagToken
+		p.tok.Attr = nil
+		return nil
+	}
+	p.tokenizer.Next()
+	p.tok = p.tokenizer.Token()
+	switch p.tok.Type {
+	case ErrorToken:
+		return p.tokenizer.Error()
+	case SelfClosingTagToken:
+		p.hasSelfClosingToken = true
+		p.tok.Type = StartTagToken
+	}
+	return nil
+}
+
+// Section 10.2.4.
+func (p *parser) acknowledgeSelfClosingTag() {
+	p.hasSelfClosingToken = false
+}
+
+// An insertion mode (section 10.2.3.1) is the state transition function from
+// a particular state in the HTML5 parser's state machine. It updates the
+// parser's fields depending on parser.token (where ErrorToken means EOF). In
+// addition to returning the next insertionMode state, it also returns whether
+// the token was consumed.
+type insertionMode func(*parser) (insertionMode, bool)
+
+// useTheRulesFor runs the delegate insertionMode over p, returning the actual
+// insertionMode unless the delegate caused a state transition.
+// Section 10.2.3.1, "using the rules for".
+func useTheRulesFor(p *parser, actual, delegate insertionMode) (insertionMode, bool) {
+	im, consumed := delegate(p)
+	if im != delegate {
+		return im, consumed
+	}
+	return actual, consumed
+}
+
+// Section 10.2.5.4.
+func initialIM(p *parser) (insertionMode, bool) {
+	// TODO: check p.tok for DOCTYPE.
+	return beforeHTMLIM, false
+}
+
+// Section 10.2.5.5.
+func beforeHTMLIM(p *parser) (insertionMode, bool) {
+	var (
+		add     bool
+		attr    []Attribute
+		implied bool
+	)
+	switch p.tok.Type {
+	case ErrorToken:
+		implied = true
+	case TextToken:
+		// TODO: distinguish whitespace text from others.
+		implied = true
+	case StartTagToken:
+		if p.tok.Data == "html" {
+			add = true
+			attr = p.tok.Attr
+		} else {
+			implied = true
+		}
+	case EndTagToken:
+		switch p.tok.Data {
+		case "head", "body", "html", "br":
+			implied = true
+		default:
+			// Ignore the token.
+		}
+	}
+	if add || implied {
+		p.addElement("html", attr)
+	}
+	return beforeHeadIM, !implied
+}
+
+// Section 10.2.5.6.
+func beforeHeadIM(p *parser) (insertionMode, bool) {
+	var (
+		add     bool
+		attr    []Attribute
+		implied bool
+	)
+	switch p.tok.Type {
+	case ErrorToken:
+		implied = true
+	case TextToken:
+		// TODO: distinguish whitespace text from others.
+		implied = true
+	case StartTagToken:
+		switch p.tok.Data {
+		case "head":
+			add = true
+			attr = p.tok.Attr
+		case "html":
+			return useTheRulesFor(p, beforeHeadIM, inBodyIM)
+		default:
+			implied = true
+		}
+	case EndTagToken:
+		switch p.tok.Data {
+		case "head", "body", "html", "br":
+			implied = true
+		default:
+			// Ignore the token.
+		}
+	}
+	if add || implied {
+		p.addElement("head", attr)
+	}
+	return inHeadIM, !implied
+}
+
+// Section 10.2.5.7.
+func inHeadIM(p *parser) (insertionMode, bool) {
+	var (
+		pop     bool
+		implied bool
+	)
+	switch p.tok.Type {
+	case ErrorToken, TextToken:
+		implied = true
+	case StartTagToken:
+		switch p.tok.Data {
+		case "meta":
+			// TODO.
+		case "script":
+			// TODO.
+		default:
+			implied = true
+		}
+	case EndTagToken:
+		if p.tok.Data == "head" {
+			pop = true
+		}
+		// TODO.
+	}
+	if pop || implied {
+		n := p.pop()
+		if n.Data != "head" {
+			panic("html: bad parser state")
+		}
+		return afterHeadIM, !implied
+	}
+	return inHeadIM, !implied
+}
+
+// Section 10.2.5.9.
+func afterHeadIM(p *parser) (insertionMode, bool) {
+	var (
+		add        bool
+		attr       []Attribute
+		framesetOK bool
+		implied    bool
+	)
+	switch p.tok.Type {
+	case ErrorToken, TextToken:
+		implied = true
+		framesetOK = true
+	case StartTagToken:
+		switch p.tok.Data {
+		case "html":
+			// TODO.
+		case "body":
+			add = true
+			attr = p.tok.Attr
+			framesetOK = false
+		case "frameset":
+			// TODO.
+		case "base", "basefont", "bgsound", "link", "meta", "noframes", "script", "style", "title":
+			// TODO.
+		case "head":
+			// TODO.
+		default:
+			implied = true
+			framesetOK = true
+		}
+	case EndTagToken:
+		// TODO.
+	}
+	if add || implied {
+		p.addElement("body", attr)
+		p.framesetOK = framesetOK
+	}
+	return inBodyIM, !implied
+}
+
+// Section 10.2.5.10.
+func inBodyIM(p *parser) (insertionMode, bool) {
+	var endP bool
+	switch p.tok.Type {
+	case TextToken:
+		p.addText(p.tok.Data)
+		p.framesetOK = false
+	case StartTagToken:
+		switch p.tok.Data {
+		case "address", "article", "aside", "blockquote", "center", "details", "dir", "div", "dl", "fieldset", "figcaption", "figure", "footer", "header", "hgroup", "menu", "nav", "ol", "p", "section", "summary", "ul":
+			// TODO: Do the proper "does the stack of open elements has a p element in button scope" algorithm in section 10.2.3.2.
+			n := p.top()
+			if n.Type == ElementNode && n.Data == "p" {
+				endP = true
+			} else {
+				p.addElement(p.tok.Data, p.tok.Attr)
+			}
+		case "h1", "h2", "h3", "h4", "h5", "h6":
+			// TODO: auto-insert 
 if necessary.
+			switch n := p.top(); n.Data {
+			case "h1", "h2", "h3", "h4", "h5", "h6":
+				p.pop()
+			}
+			p.addElement(p.tok.Data, p.tok.Attr)
+		case "b", "big", "code", "em", "font", "i", "s", "small", "strike", "strong", "tt", "u":
+			p.reconstructActiveFormattingElements()
+			p.addFormattingElement(p.tok.Data, p.tok.Attr)
+		case "area", "br", "embed", "img", "input", "keygen", "wbr":
+			p.reconstructActiveFormattingElements()
+			p.addElement(p.tok.Data, p.tok.Attr)
+			p.pop()
+			p.acknowledgeSelfClosingTag()
+			p.framesetOK = false
+		case "table":
+			// TODO: auto-insert 
 if necessary, depending on quirks mode.
+			p.addElement(p.tok.Data, p.tok.Attr)
+			p.framesetOK = false
+			return inTableIM, true
+		case "hr":
+			// TODO: auto-insert 
 if necessary.
+			p.addElement(p.tok.Data, p.tok.Attr)
+			p.pop()
+			p.acknowledgeSelfClosingTag()
+			p.framesetOK = false
+		default:
+			// TODO.
+		}
+	case EndTagToken:
+		switch p.tok.Data {
+		case "body":
+			// TODO: autoclose the stack of open elements.
+			return afterBodyIM, true
+		case "a", "b", "big", "code", "em", "font", "i", "nobr", "s", "small", "strike", "strong", "tt", "u":
+			// TODO: implement the "adoption agency" algorithm:
+			// http://www.whatwg.org/specs/web-apps/current-work/multipage/tokenization.html#adoptionAgency
+			if p.tok.Data == p.top().Data {
+				p.pop()
+			}
+		default:
+			// TODO.
+		}
+	}
+	if endP {
+		// TODO: do the proper algorithm.
+		n := p.pop()
+		if n.Type != ElementNode || n.Data != "p" {
+			panic("unreachable")
+		}
+	}
+	return inBodyIM, !endP
+}
+
+// Section 10.2.5.12.
+func inTableIM(p *parser) (insertionMode, bool) {
+	var (
+		add      bool
+		data     string
+		attr     []Attribute
+		consumed bool
+	)
+	switch p.tok.Type {
+	case ErrorToken:
+		// Stop parsing.
+		return nil, true
+	case TextToken:
+		// TODO.
+	case StartTagToken:
+		switch p.tok.Data {
+		case "tbody", "tfoot", "thead":
+			add = true
+			data = p.tok.Data
+			attr = p.tok.Attr
+			consumed = true
+		case "td", "th", "tr":
+			add = true
+			data = "tbody"
+		default:
+			// TODO.
+		}
+	case EndTagToken:
+		switch p.tok.Data {
+		case "table":
+			if p.popUntil(tableScopeStopTags, "table") {
+				// TODO: "reset the insertion mode appropriately" as per 10.2.3.1.
+				return inBodyIM, false
+			}
+			// Ignore the token.
+			return inTableIM, true
+		case "body", "caption", "col", "colgroup", "html", "tbody", "td", "tfoot", "th", "thead", "tr":
+			// Ignore the token.
+			return inTableIM, true
+		}
+	}
+	if add {
+		// TODO: clear the stack back to a table context.
+		p.addElement(data, attr)
+		return inTableBodyIM, consumed
+	}
+	// TODO: return useTheRulesFor(inTableIM, inBodyIM, p) unless etc. etc. foster parenting.
+	return inTableIM, true
+}
+
+// Section 10.2.5.16.
+func inTableBodyIM(p *parser) (insertionMode, bool) {
+	var (
+		add      bool
+		data     string
+		attr     []Attribute
+		consumed bool
+	)
+	switch p.tok.Type {
+	case ErrorToken:
+		// TODO.
+	case TextToken:
+		// TODO.
+	case StartTagToken:
+		switch p.tok.Data {
+		case "tr":
+			add = true
+			data = p.tok.Data
+			attr = p.tok.Attr
+			consumed = true
+		case "td", "th":
+			add = true
+			data = "tr"
+			consumed = false
+		default:
+			// TODO.
+		}
+	case EndTagToken:
+		switch p.tok.Data {
+		case "table":
+			if p.popUntil(tableScopeStopTags, "tbody", "thead", "tfoot") {
+				return inTableIM, false
+			}
+			// Ignore the token.
+			return inTableBodyIM, true
+		case "body", "caption", "col", "colgroup", "html", "td", "th", "tr":
+			// Ignore the token.
+			return inTableBodyIM, true
+		}
+	}
+	if add {
+		// TODO: clear the stack back to a table body context.
+		p.addElement(data, attr)
+		return inRowIM, consumed
+	}
+	return useTheRulesFor(p, inTableBodyIM, inTableIM)
+}
+
+// Section 10.2.5.17.
+func inRowIM(p *parser) (insertionMode, bool) {
+	switch p.tok.Type {
+	case ErrorToken:
+		// TODO.
+	case TextToken:
+		// TODO.
+	case StartTagToken:
+		switch p.tok.Data {
+		case "td", "th":
+			// TODO: clear the stack back to a table row context.
+			p.addElement(p.tok.Data, p.tok.Attr)
+			// TODO: insert a marker at the end of the list of active formatting elements.
+			return inCellIM, true
+		default:
+			// TODO.
+		}
+	case EndTagToken:
+		switch p.tok.Data {
+		case "tr":
+			// TODO.
+		case "table":
+			if p.popUntil(tableScopeStopTags, "tr") {
+				return inTableBodyIM, false
+			}
+			// Ignore the token.
+			return inRowIM, true
+		case "tbody", "tfoot", "thead":
+			// TODO.
+		case "body", "caption", "col", "colgroup", "html", "td", "th":
+			// Ignore the token.
+			return inRowIM, true
+		default:
+			// TODO.
+		}
+	}
+	return useTheRulesFor(p, inRowIM, inTableIM)
+}
+
+// Section 10.2.5.18.
+func inCellIM(p *parser) (insertionMode, bool) {
+	var (
+		closeTheCellAndReprocess bool
+	)
+	switch p.tok.Type {
+	case StartTagToken:
+		switch p.tok.Data {
+		case "caption", "col", "colgroup", "tbody", "td", "tfoot", "th", "thead", "tr":
+			// TODO: check for "td" or "th" in table scope.
+			closeTheCellAndReprocess = true
+		}
+	case EndTagToken:
+		switch p.tok.Data {
+		case "td", "th":
+			// TODO.
+		case "body", "caption", "col", "colgroup", "html":
+			// TODO.
+		case "table", "tbody", "tfoot", "thead", "tr":
+			// TODO: check for matching element in table scope.
+			closeTheCellAndReprocess = true
+		}
+	}
+	if closeTheCellAndReprocess {
+		if p.popUntil(tableScopeStopTags, "td") || p.popUntil(tableScopeStopTags, "th") {
+			// TODO: clear the list of active formatting elements up to the last marker.
+			return inRowIM, false
+		}
+	}
+	return useTheRulesFor(p, inCellIM, inBodyIM)
+}
+
+// Section 10.2.5.22.
+func afterBodyIM(p *parser) (insertionMode, bool) {
+	switch p.tok.Type {
+	case ErrorToken:
+		// TODO.
+	case TextToken:
+		// TODO.
+	case StartTagToken:
+		// TODO.
+	case EndTagToken:
+		switch p.tok.Data {
+		case "html":
+			// TODO: autoclose the stack of open elements.
+			return afterAfterBodyIM, true
+		default:
+			// TODO.
+		}
+	}
+	return afterBodyIM, true
+}
+
+// Section 10.2.5.25.
+func afterAfterBodyIM(p *parser) (insertionMode, bool) {
+	switch p.tok.Type {
+	case ErrorToken:
+		// Stop parsing.
+		return nil, true
+	case TextToken:
+		// TODO.
+	case StartTagToken:
+		if p.tok.Data == "html" {
+			return useTheRulesFor(p, afterAfterBodyIM, inBodyIM)
+		}
+	}
+	return inBodyIM, false
+}
+
+// Parse returns the parse tree for the HTML from the given Reader.
+// The input is assumed to be UTF-8 encoded.
+func Parse(r io.Reader) (*Node, os.Error) {
+	p := &parser{
+		tokenizer: NewTokenizer(r),
+		doc: &Node{
+			Type: DocumentNode,
+		},
+		scripting:  true,
+		framesetOK: true,
+	}
+	// Iterate until EOF. Any other error will cause an early return.
+	im, consumed := initialIM, true
+	for {
+		if consumed {
+			if err := p.read(); err != nil {
+				if err == os.EOF {
+					break
+				}
+				return nil, err
+			}
+		}
+		im, consumed = im(p)
+	}
+	// Loop until the final token (the ErrorToken signifying EOF) is consumed.
+	for {
+		if im, consumed = im(p); consumed {
+			break
+		}
+	}
+	return p.doc, nil
+}
diff --git a/libgo/go/html/parse_test.go b/libgo/go/html/parse_test.go
new file mode 100644
index 000000000..d153533b5
--- /dev/null
+++ b/libgo/go/html/parse_test.go
@@ -0,0 +1,158 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package html
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"strings"
+	"testing"
+)
+
+type devNull struct{}
+
+func (devNull) Write(p []byte) (int, os.Error) {
+	return len(p), nil
+}
+
+func pipeErr(err os.Error) io.Reader {
+	pr, pw := io.Pipe()
+	pw.CloseWithError(err)
+	return pr
+}
+
+func readDat(filename string, c chan io.Reader) {
+	f, err := os.Open("testdata/webkit/"+filename, os.O_RDONLY, 0600)
+	if err != nil {
+		c <- pipeErr(err)
+		return
+	}
+	defer f.Close()
+
+	// Loop through the lines of the file. Each line beginning with "#" denotes
+	// a new section, which is returned as a separate io.Reader.
+	r := bufio.NewReader(f)
+	var pw *io.PipeWriter
+	for {
+		line, err := r.ReadSlice('\n')
+		if err != nil {
+			if pw != nil {
+				pw.CloseWithError(err)
+				pw = nil
+			} else {
+				c <- pipeErr(err)
+			}
+			return
+		}
+		if len(line) == 0 {
+			continue
+		}
+		if line[0] == '#' {
+			if pw != nil {
+				pw.Close()
+			}
+			var pr *io.PipeReader
+			pr, pw = io.Pipe()
+			c <- pr
+			continue
+		}
+		if line[0] != '|' {
+			// Strip the trailing '\n'.
+			line = line[:len(line)-1]
+		}
+		if pw != nil {
+			if _, err := pw.Write(line); err != nil {
+				pw.CloseWithError(err)
+				pw = nil
+			}
+		}
+	}
+}
+
+func dumpLevel(w io.Writer, n *Node, level int) os.Error {
+	io.WriteString(w, "| ")
+	for i := 0; i < level; i++ {
+		io.WriteString(w, "  ")
+	}
+	switch n.Type {
+	case ErrorNode:
+		return os.NewError("unexpected ErrorNode")
+	case DocumentNode:
+		return os.NewError("unexpected DocumentNode")
+	case ElementNode:
+		fmt.Fprintf(w, "<%s>", EscapeString(n.Data))
+	case TextNode:
+		fmt.Fprintf(w, "%q", EscapeString(n.Data))
+	case CommentNode:
+		return os.NewError("COMMENT")
+	default:
+		return os.NewError("unknown node type")
+	}
+	io.WriteString(w, "\n")
+	for _, c := range n.Child {
+		if err := dumpLevel(w, c, level+1); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func dump(n *Node) (string, os.Error) {
+	if n == nil || len(n.Child) == 0 {
+		return "", nil
+	}
+	b := bytes.NewBuffer(nil)
+	for _, child := range n.Child {
+		if err := dumpLevel(b, child, 0); err != nil {
+			return "", err
+		}
+	}
+	return b.String(), nil
+}
+
+func TestParser(t *testing.T) {
+	// TODO(nigeltao): Process all the .dat files, not just the first one.
+	filenames := []string{
+		"tests1.dat",
+	}
+	for _, filename := range filenames {
+		rc := make(chan io.Reader)
+		go readDat(filename, rc)
+		// TODO(nigeltao): Process all test cases, not just a subset.
+		for i := 0; i < 22; i++ {
+			// Parse the #data section.
+			b, err := ioutil.ReadAll(<-rc)
+			if err != nil {
+				t.Fatal(err)
+			}
+			text := string(b)
+			doc, err := Parse(strings.NewReader(text))
+			if err != nil {
+				t.Fatal(err)
+			}
+			actual, err := dump(doc)
+			if err != nil {
+				t.Fatal(err)
+			}
+			// Skip the #error section.
+			if _, err := io.Copy(devNull{}, <-rc); err != nil {
+				t.Fatal(err)
+			}
+			// Compare the parsed tree to the #document section.
+			b, err = ioutil.ReadAll(<-rc)
+			if err != nil {
+				t.Fatal(err)
+			}
+			expected := string(b)
+			if actual != expected {
+				t.Errorf("%s test #%d %q, actual vs expected:\n----\n%s----\n%s----", filename, i, text, actual, expected)
+			}
+		}
+	}
+}
diff --git a/libgo/go/html/testdata/webkit/README b/libgo/go/html/testdata/webkit/README
new file mode 100644
index 000000000..9b4c2d8be
--- /dev/null
+++ b/libgo/go/html/testdata/webkit/README
@@ -0,0 +1,28 @@
+The *.dat files in this directory are copied from The WebKit Open Source
+Project, specifically $WEBKITROOT/LayoutTests/html5lib/resources.
+WebKit is licensed under a BSD style license.
+http://webkit.org/coding/bsd-license.html says:
+
+Copyright (C) 2009 Apple Inc. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS "AS IS" AND ANY
+EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
diff --git a/libgo/go/html/testdata/webkit/comments01.dat b/libgo/go/html/testdata/webkit/comments01.dat
new file mode 100644
index 000000000..388d95287
--- /dev/null
+++ b/libgo/go/html/testdata/webkit/comments01.dat
@@ -0,0 +1,126 @@
+#data
+FOOBAZ
+#errors
+#document
+| 
+|   
+|   
+|     "FOO"
+|     
+|     "BAZ"
+
+#data
+FOOBAZ
+#errors
+#document
+| 
+|   
+|   
+|     "FOO"
+|     
+|     "BAZ"
+
+#data
+FOO
+|     "BAZ"
+
+#data
+FOOBAZ
+#errors
+#document
+| 
+|   
+|   
+|     "FOO"
+|     
+|     "BAZ"
+
+#data
+FOOBAZ
+#errors
+#document
+| 
+|   
+|   
+|     "FOO"
+|     
+|     "BAZ"
+
+#data
+FOO
+|     "BAZ"
+
+#data
+FOOBAZ
+#errors
+#document
+| 
+|   
+|   
+|     "FOO"
+|     
+|     "BAZ"
+
+#data
+FOOBAZ
+#errors
+#document
+| 
+|   
+|   
+|     "FOO"
+|     
+|     "BAZ"
+
+#data
+FOOBAZ
+#errors
+#document
+| 
+|   
+|   
+|     "FOO"
+|     
+|     "BAZ"
+
+#data
+Hi
+#errors
+#document
+| 
+| 
+|   
+|   
+|     "Hi"
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+
+#data
+
+| 
+|   
+|   
diff --git a/libgo/go/html/testdata/webkit/doctype01.dat b/libgo/go/html/testdata/webkit/doctype01.dat
new file mode 100644
index 000000000..575129c14
--- /dev/null
+++ b/libgo/go/html/testdata/webkit/doctype01.dat
@@ -0,0 +1,335 @@
+#data
+Hello
+#errors
+#document
+| 
+| 
+|   
+|   
+|     "Hello"
+
+#data
+Hello
+#errors
+#document
+| 
+| 
+|   
+|   
+|     "Hello"
+
+#data
+Hello
+#errors
+#document
+| 
+| 
+|   
+|   
+|     "Hello"
+
+#data
+Hello
+#errors
+#document
+| 
+| 
+|   
+|   
+|     "Hello"
+
+#data
+Hello
+#errors
+#document
+| 
+| 
+|   
+|   
+|     "Hello"
+
+#data
+Hello
+#errors
+#document
+| 
+| 
+|   
+|   
+|     "Hello"
+
+#data
+Hello
+#errors
+#document
+| 
+| 
+|   
+|   
+|     "Hello"
+
+#data
+Hello
+#errors
+#document
+| 
+| 
+|   
+|   
+|     "Hello"
+
+#data
+Hello
+#errors
+#document
+| 
+| 
+|   
+|   
+|     "Hello"
+
+#data
+Hello
+#errors
+#document
+| 
+| 
+|   
+|   
+|     "Hello"
+
+#data
+Hello
+#errors
+#document
+| 
+| 
+|   
+|   
+|     "Hello"
+
+#data
+Hello
+#errors
+#document
+| 
+| 
+|   
+|   
+|     "Hello"
+
+#data
+Hello
+#errors
+#document
+| 
+| 
+|   
+|   
+|     "Hello"
+
+#data
+Hello
+#errors
+#document
+| 
+| 
+|   
+|   
+|     "Hello"
+
+#data
+Hello
+#errors
+#document
+| 
+| 
+|   
+|   
+|     "Hello"
+
+#data
+Hello
+#errors
+#document
+| 
+| 
+|   
+|   
+|     "Hello"
+
+#data
+Hello
+#errors
+#document
+| 
+| 
+|   
+|   
+|     "Hello"
+
+#data
+Hello
+#errors
+#document
+| 
+| 
+|   
+|   
+|     "Hello"
+
+#data
+Hello
+#errors
+#document
+| 
+| 
+|   
+|   
+|     "Hello"
+
+#data
+Hello
+#errors
+#document
+| 
+| 
+|   
+|   
+|     "Hello"
+
+#data
+Hello
+#errors
+#document
+| 
+| 
+|   
+|   
+|     "Hello"
+
+#data
+Hello
+#errors
+#document
+| 
+| 
+|   
+|   
+|     "Hello"
+
+#data
+Hello
+#errors
+#document
+| 
+| 
+|   
+|   
+|     "Hello"
+
+#data
+Hello
+#errors
+#document
+| 
+| 
+|   
+|   
+|     "Hello"
+
+#data
+Hello
+#errors
+#document
+| 
+| 
+|   
+|   
+|     "Hello"
+
+#data
+Hello
+#errors
+#document
+| 
+| 
+|   
+|   
+|     "Hello"
+
+#data
+Hello
+#errors
+#document
+| 
+| 
+|   
+|   
+|     "Hello"
+
+#data
+Hello
+#errors
+#document
+| 
+| 
+|   
+|   
+|     "Hello"
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+
+#data
+
+]>
+#errors
+#document
+| 
+| 
+|   
+|   
+|     "
+]>"
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+
+#data
+Mine!
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       "Mine!"
diff --git a/libgo/go/html/testdata/webkit/dom2string.js b/libgo/go/html/testdata/webkit/dom2string.js
new file mode 100644
index 000000000..45897fda4
--- /dev/null
+++ b/libgo/go/html/testdata/webkit/dom2string.js
@@ -0,0 +1,135 @@
+String.prototype.toAsciiLowerCase = function () {
+  var output = "";
+  for (var i = 0, len = this.length; i < len; ++i) {
+    if (this.charCodeAt(i) >= 0x41 && this.charCodeAt(i) <= 0x5A) {
+      output += String.fromCharCode(this.charCodeAt(i) + 0x20)
+    } else {
+      output += this.charAt(i);
+    }
+  }
+  return output;
+}
+
+function indent(ancestors) {
+  var str = "";
+  if (ancestors > 0) {
+    while (ancestors--)
+      str += "  ";
+  }
+  return str;
+}
+
+function dom2string(node, ancestors) {
+  var str = "";
+  if (typeof ancestors == "undefined")
+    var ancestors = 0;
+  if (!node.firstChild)
+    return "| ";
+  var parent = node;
+  var current = node.firstChild;
+  var next = null;
+  var misnested = null;
+  for (;;) {
+    str += "\n| " + indent(ancestors);
+    switch (current.nodeType) {
+      case 10:
+        str += '';
+        break;
+      case 8:
+        try {
+          str += '';
+        } catch (e) {
+          str += '';
+        }
+        if (parent != current.parentNode) {
+          return str += ' (misnested... aborting)';
+        }
+        break;
+      case 7:
+        str += '';
+        break;
+      case 4:
+        str += '';
+        break;
+      case 3:
+        str += '"' + current.nodeValue + '"';
+        if (parent != current.parentNode) {
+          return str += ' (misnested... aborting)';
+        }
+        break;
+      case 1:
+        str += "<";
+        switch (current.namespaceURI) {
+          case "http://www.w3.org/2000/svg":
+            str += "svg ";
+            break;
+          case "http://www.w3.org/1998/Math/MathML":
+            str += "math ";
+            break;
+        }
+        if (current.localName && current.namespaceURI && current.namespaceURI != null) {
+          str += current.localName;
+        } else {
+          str += current.nodeName.toAsciiLowerCase();
+        }
+        str += '>';
+        if (parent != current.parentNode) {
+          return str += ' (misnested... aborting)';
+        } else {
+          if (current.attributes) {
+            var attrNames = [];
+            var attrPos = {};
+            for (var j = 0; j < current.attributes.length; j += 1) {
+              if (current.attributes[j].specified) {
+                var name = "";
+                switch (current.attributes[j].namespaceURI) {
+                  case "http://www.w3.org/XML/1998/namespace":
+                    name += "xml ";
+                    break;
+                  case "http://www.w3.org/2000/xmlns/":
+                    name += "xmlns ";
+                    break;
+                  case "http://www.w3.org/1999/xlink":
+                    name += "xlink ";
+                    break;
+                }
+                if (current.attributes[j].localName) {
+                  name += current.attributes[j].localName;
+                } else {
+                  name += current.attributes[j].nodeName;
+                }
+                attrNames.push(name);
+                attrPos[name] = j;
+              }
+            }
+            if (attrNames.length > 0) {
+              attrNames.sort();
+              for (var j = 0; j < attrNames.length; j += 1) {
+                str += "\n| " + indent(1 + ancestors) + attrNames[j];
+                str += '="' + current.attributes[attrPos[attrNames[j]]].nodeValue + '"';
+              }
+            }
+          }
+          if (next = current.firstChild) {
+            parent = current;
+            current = next;
+            ancestors++;
+            continue;
+          }
+        }
+        break;
+    }
+    for (;;) {
+      if (next = current.nextSibling) {
+        current = next;
+        break;
+      }
+      current = current.parentNode;
+      parent = parent.parentNode;
+      ancestors--;
+      if (current == node) {
+        return str.substring(1);
+      }
+    }
+  }
+}
diff --git a/libgo/go/html/testdata/webkit/entities01.dat b/libgo/go/html/testdata/webkit/entities01.dat
new file mode 100644
index 000000000..926642e2e
--- /dev/null
+++ b/libgo/go/html/testdata/webkit/entities01.dat
@@ -0,0 +1,612 @@
+#data
+FOO>BAR
+#errors
+#document
+| 
+|   
+|   
+|     "FOO>BAR"
+
+#data
+FOO>BAR
+#errors
+#document
+| 
+|   
+|   
+|     "FOO>BAR"
+
+#data
+FOO> BAR
+#errors
+#document
+| 
+|   
+|   
+|     "FOO> BAR"
+
+#data
+FOO>;;BAR
+#errors
+#document
+| 
+|   
+|   
+|     "FOO>;;BAR"
+
+#data
+I'm ¬it; I tell you
+#errors
+#document
+| 
+|   
+|   
+|     "I'm ¬it; I tell you"
+
+#data
+I'm ∉ I tell you
+#errors
+#document
+| 
+|   
+|   
+|     "I'm ∉ I tell you"
+
+#data
+FOO& BAR
+#errors
+#document
+| 
+|   
+|   
+|     "FOO& BAR"
+
+#data
+FOO&
+#errors
+#document
+| 
+|   
+|   
+|     "FOO&"
+|     
+
+#data
+FOO&&&>BAR
+#errors
+#document
+| 
+|   
+|   
+|     "FOO&&&>BAR"
+
+#data
+FOO)BAR
+#errors
+#document
+| 
+|   
+|   
+|     "FOO)BAR"
+
+#data
+FOOABAR
+#errors
+#document
+| 
+|   
+|   
+|     "FOOABAR"
+
+#data
+FOOABAR
+#errors
+#document
+| 
+|   
+|   
+|     "FOOABAR"
+
+#data
+FOO&#BAR
+#errors
+#document
+| 
+|   
+|   
+|     "FOO&#BAR"
+
+#data
+FOO&#ZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOO&#ZOO"
+
+#data
+FOOºR
+#errors
+#document
+| 
+|   
+|   
+|     "FOOºR"
+
+#data
+FOO&#xZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOO&#xZOO"
+
+#data
+FOO&#XZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOO&#XZOO"
+
+#data
+FOO)BAR
+#errors
+#document
+| 
+|   
+|   
+|     "FOO)BAR"
+
+#data
+FOO䆺R
+#errors
+#document
+| 
+|   
+|   
+|     "FOO䆺R"
+
+#data
+FOOAZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOOAZOO"
+
+#data
+FOO�ZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOO�ZOO"
+
+#data
+FOO
ZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOO
ZOO"
+
+#data
+FOOxZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOOxZOO"
+
+#data
+FOOyZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOOyZOO"
+
+#data
+FOO€ZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOO€ZOO"
+
+#data
+FOOZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOOZOO"
+
+#data
+FOO‚ZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOO‚ZOO"
+
+#data
+FOOƒZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOOƒZOO"
+
+#data
+FOO„ZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOO„ZOO"
+
+#data
+FOO…ZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOO…ZOO"
+
+#data
+FOO†ZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOO†ZOO"
+
+#data
+FOO‡ZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOO‡ZOO"
+
+#data
+FOOˆZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOOˆZOO"
+
+#data
+FOO‰ZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOO‰ZOO"
+
+#data
+FOOŠZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOOŠZOO"
+
+#data
+FOO‹ZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOO‹ZOO"
+
+#data
+FOOŒZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOOŒZOO"
+
+#data
+FOOZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOOZOO"
+
+#data
+FOOŽZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOOŽZOO"
+
+#data
+FOOZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOOZOO"
+
+#data
+FOOZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOOZOO"
+
+#data
+FOO‘ZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOO‘ZOO"
+
+#data
+FOO’ZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOO’ZOO"
+
+#data
+FOO“ZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOO“ZOO"
+
+#data
+FOO”ZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOO”ZOO"
+
+#data
+FOO•ZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOO•ZOO"
+
+#data
+FOO–ZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOO–ZOO"
+
+#data
+FOO—ZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOO—ZOO"
+
+#data
+FOO˜ZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOO˜ZOO"
+
+#data
+FOO™ZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOO™ZOO"
+
+#data
+FOOšZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOOšZOO"
+
+#data
+FOO›ZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOO›ZOO"
+
+#data
+FOOœZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOOœZOO"
+
+#data
+FOOZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOOZOO"
+
+#data
+FOOžZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOOžZOO"
+
+#data
+FOOŸZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOOŸZOO"
+
+#data
+FOO ZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOO ZOO"
+
+#data
+FOO퟿ZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOO퟿ZOO"
+
+#data
+FOO�ZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOO�ZOO"
+
+#data
+FOO�ZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOO�ZOO"
+
+#data
+FOO�ZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOO�ZOO"
+
+#data
+FOO�ZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOO�ZOO"
+
+#data
+FOOZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOOZOO"
+
+#data
+FOO􏿾ZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOO􏿾ZOO"
+
+#data
+FOO􈟔ZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOO􈟔ZOO"
+
+#data
+FOO􏿿ZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOO􏿿ZOO"
+
+#data
+FOO�ZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOO�ZOO"
+
+#data
+FOO�ZOO
+#errors
+#document
+| 
+|   
+|   
+|     "FOO�ZOO"
diff --git a/libgo/go/html/testdata/webkit/entities02.dat b/libgo/go/html/testdata/webkit/entities02.dat
new file mode 100644
index 000000000..0b4dd6681
--- /dev/null
+++ b/libgo/go/html/testdata/webkit/entities02.dat
@@ -0,0 +1,129 @@
+#data
+

+#errors
+#document
+| 
+|   
+|   
+|     

+|       bar="ZZ>YY"
+
+#data
+

+#errors
+#document
+| 
+|   
+|   
+|     

+|       bar="ZZ&"
+
+#data
+

+#errors
+#document
+| 
+|   
+|   
+|     

+|       bar="ZZ&"
+
+#data
+

+#errors
+#document
+| 
+|   
+|   
+|     

+|       bar="ZZ&"
+
+#data
+

+#errors
+#document
+| 
+|   
+|   
+|     

+|       bar="ZZ>=YY"
+
+#data
+

+#errors
+#document
+| 
+|   
+|   
+|     

+|       bar="ZZ>0YY"
+
+#data
+

+#errors
+#document
+| 
+|   
+|   
+|     

+|       bar="ZZ>9YY"
+
+#data
+

+#errors
+#document
+| 
+|   
+|   
+|     

+|       bar="ZZ>aYY"
+
+#data
+

+#errors
+#document
+| 
+|   
+|   
+|     

+|       bar="ZZ>ZYY"
+
+#data
+

+#errors
+#document
+| 
+|   
+|   
+|     

+|       bar="ZZ> YY"
+
+#data
+

+#errors
+#document
+| 
+|   
+|   
+|     

+|       bar="ZZ>"
+
+#data
+

+#errors
+#document
+| 
+|   
+|   
+|     

+|       bar="ZZ>"
+
+#data
+

+#errors
+#document
+| 
+|   
+|   
+|     

+|       bar="ZZ>"
diff --git a/libgo/go/html/testdata/webkit/scriptdata01.dat b/libgo/go/html/testdata/webkit/scriptdata01.dat
new file mode 100644
index 000000000..76b67f4ba
--- /dev/null
+++ b/libgo/go/html/testdata/webkit/scriptdata01.dat
@@ -0,0 +1,308 @@
+#data
+FOOBAR
+#errors
+#document
+| 
+|   
+|   
+|     "FOO"
+|     BAR
+#errors
+#document
+| 
+|   
+|   
+|     "FOO"
+|     BAR
+#errors
+#document
+| 
+|   
+|   
+|     "FOO"
+|     BAR
+#errors
+#document
+| 
+|   
+|   
+|     "FOO"
+|     BAR
+#errors
+#document
+| 
+|   
+|   
+|     "FOO"
+|     BAR
+#errors
+#document
+| 
+|   
+|   
+|     "FOO"
+|     BAR
+#errors
+#document
+| 
+|   
+|   
+|     "FOO"
+|     BAR
+#errors
+#document
+| 
+|   
+|   
+|     "FOO"
+|     BAR
+#errors
+#document
+| 
+|   
+|   
+|     "FOO"
+|     BAR
+#errors
+#document
+| 
+|   
+|   
+|     "FOO"
+|     BAR
+#errors
+#document
+| 
+|   
+|   
+|     "FOO"
+|     BAR
+#errors
+#document
+| 
+|   
+|   
+|     "FOO"
+|     BAR
+#errors
+#document
+| 
+|   
+|   
+|     "FOO"
+|     BAR
+#errors
+#document
+| 
+|   
+|   
+|     "FOO"
+|     QUX
+#errors
+#document
+| 
+|   
+|   
+|     "FOO"
+|      -->  EOF
+#errors
+#document
+| 
+| 
+|   
+|     
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+Line: 1 Col: 51 Unexpected end of file. Expected end tag (style).
+#document
+| 
+|   
+|     

+#errors
+Line: 1 Col: 9 Unexpected end tag (strong). Expected DOCTYPE.
+Line: 1 Col: 9 Unexpected end tag (strong) after the (implied) root element.
+Line: 1 Col: 13 Unexpected end tag (b) after the (implied) root element.
+Line: 1 Col: 18 Unexpected end tag (em) after the (implied) root element.
+Line: 1 Col: 22 Unexpected end tag (i) after the (implied) root element.
+Line: 1 Col: 26 Unexpected end tag (u) after the (implied) root element.
+Line: 1 Col: 35 Unexpected end tag (strike) after the (implied) root element.
+Line: 1 Col: 39 Unexpected end tag (s) after the (implied) root element.
+Line: 1 Col: 47 Unexpected end tag (blink) after the (implied) root element.
+Line: 1 Col: 52 Unexpected end tag (tt) after the (implied) root element.
+Line: 1 Col: 58 Unexpected end tag (pre) after the (implied) root element.
+Line: 1 Col: 64 Unexpected end tag (big) after the (implied) root element.
+Line: 1 Col: 72 Unexpected end tag (small) after the (implied) root element.
+Line: 1 Col: 79 Unexpected end tag (font) after the (implied) root element.
+Line: 1 Col: 88 Unexpected end tag (select) after the (implied) root element.
+Line: 1 Col: 93 Unexpected end tag (h1) after the (implied) root element.
+Line: 1 Col: 98 Unexpected end tag (h2) after the (implied) root element.
+Line: 1 Col: 103 Unexpected end tag (h3) after the (implied) root element.
+Line: 1 Col: 108 Unexpected end tag (h4) after the (implied) root element.
+Line: 1 Col: 113 Unexpected end tag (h5) after the (implied) root element.
+Line: 1 Col: 118 Unexpected end tag (h6) after the (implied) root element.
+Line: 1 Col: 125 Unexpected end tag (body) after the (implied) root element.
+Line: 1 Col: 130 Unexpected end tag (br). Treated as br element.
+Line: 1 Col: 134 End tag (a) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 140 This element (img) has no end tag.
+Line: 1 Col: 148 Unexpected end tag (title). Ignored.
+Line: 1 Col: 155 Unexpected end tag (span). Ignored.
+Line: 1 Col: 163 Unexpected end tag (style). Ignored.
+Line: 1 Col: 172 Unexpected end tag (script). Ignored.
+Line: 1 Col: 180 Unexpected end tag (table). Ignored.
+Line: 1 Col: 185 Unexpected end tag (th). Ignored.
+Line: 1 Col: 190 Unexpected end tag (td). Ignored.
+Line: 1 Col: 195 Unexpected end tag (tr). Ignored.
+Line: 1 Col: 203 This element (frame) has no end tag.
+Line: 1 Col: 210 This element (area) has no end tag.
+Line: 1 Col: 217 Unexpected end tag (link). Ignored.
+Line: 1 Col: 225 This element (param) has no end tag.
+Line: 1 Col: 230 This element (hr) has no end tag.
+Line: 1 Col: 238 This element (input) has no end tag.
+Line: 1 Col: 244 Unexpected end tag (col). Ignored.
+Line: 1 Col: 251 Unexpected end tag (base). Ignored.
+Line: 1 Col: 258 Unexpected end tag (meta). Ignored.
+Line: 1 Col: 269 This element (basefont) has no end tag.
+Line: 1 Col: 279 This element (bgsound) has no end tag.
+Line: 1 Col: 287 This element (embed) has no end tag.
+Line: 1 Col: 296 This element (spacer) has no end tag.
+Line: 1 Col: 300 Unexpected end tag (p). Ignored.
+Line: 1 Col: 305 End tag (dd) seen too early. Expected other end tag.
+Line: 1 Col: 310 End tag (dt) seen too early. Expected other end tag.
+Line: 1 Col: 320 Unexpected end tag (caption). Ignored.
+Line: 1 Col: 331 Unexpected end tag (colgroup). Ignored.
+Line: 1 Col: 339 Unexpected end tag (tbody). Ignored.
+Line: 1 Col: 347 Unexpected end tag (tfoot). Ignored.
+Line: 1 Col: 355 Unexpected end tag (thead). Ignored.
+Line: 1 Col: 365 End tag (address) seen too early. Expected other end tag.
+Line: 1 Col: 378 End tag (blockquote) seen too early. Expected other end tag.
+Line: 1 Col: 387 End tag (center) seen too early. Expected other end tag.
+Line: 1 Col: 393 Unexpected end tag (dir). Ignored.
+Line: 1 Col: 399 End tag (div) seen too early. Expected other end tag.
+Line: 1 Col: 404 End tag (dl) seen too early. Expected other end tag.
+Line: 1 Col: 415 End tag (fieldset) seen too early. Expected other end tag.
+Line: 1 Col: 425 End tag (listing) seen too early. Expected other end tag.
+Line: 1 Col: 432 End tag (menu) seen too early. Expected other end tag.
+Line: 1 Col: 437 End tag (ol) seen too early. Expected other end tag.
+Line: 1 Col: 442 End tag (ul) seen too early. Expected other end tag.
+Line: 1 Col: 447 End tag (li) seen too early. Expected other end tag.
+Line: 1 Col: 454 End tag (nobr) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 460 This element (wbr) has no end tag.
+Line: 1 Col: 476 End tag (button) seen too early. Expected other end tag.
+Line: 1 Col: 486 End tag (marquee) seen too early. Expected other end tag.
+Line: 1 Col: 495 End tag (object) seen too early. Expected other end tag.
+Line: 1 Col: 513 Unexpected end tag (html). Ignored.
+Line: 1 Col: 513 Unexpected end tag (frameset). Ignored.
+Line: 1 Col: 520 Unexpected end tag (head). Ignored.
+Line: 1 Col: 529 Unexpected end tag (iframe). Ignored.
+Line: 1 Col: 537 This element (image) has no end tag.
+Line: 1 Col: 547 This element (isindex) has no end tag.
+Line: 1 Col: 557 Unexpected end tag (noembed). Ignored.
+Line: 1 Col: 568 Unexpected end tag (noframes). Ignored.
+Line: 1 Col: 579 Unexpected end tag (noscript). Ignored.
+Line: 1 Col: 590 Unexpected end tag (optgroup). Ignored.
+Line: 1 Col: 599 Unexpected end tag (option). Ignored.
+Line: 1 Col: 611 Unexpected end tag (plaintext). Ignored.
+Line: 1 Col: 622 Unexpected end tag (textarea). Ignored.
+#document
+| 
+|   
+|   
+|     

+|     

+
+#data
+

+#errors
+Line: 1 Col: 7 Unexpected start tag (table). Expected DOCTYPE.
+Line: 1 Col: 20 Unexpected end tag (strong) in table context caused voodoo mode.
+Line: 1 Col: 20 End tag (strong) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 24 Unexpected end tag (b) in table context caused voodoo mode.
+Line: 1 Col: 24 End tag (b) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 29 Unexpected end tag (em) in table context caused voodoo mode.
+Line: 1 Col: 29 End tag (em) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 33 Unexpected end tag (i) in table context caused voodoo mode.
+Line: 1 Col: 33 End tag (i) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 37 Unexpected end tag (u) in table context caused voodoo mode.
+Line: 1 Col: 37 End tag (u) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 46 Unexpected end tag (strike) in table context caused voodoo mode.
+Line: 1 Col: 46 End tag (strike) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 50 Unexpected end tag (s) in table context caused voodoo mode.
+Line: 1 Col: 50 End tag (s) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 58 Unexpected end tag (blink) in table context caused voodoo mode.
+Line: 1 Col: 58 Unexpected end tag (blink). Ignored.
+Line: 1 Col: 63 Unexpected end tag (tt) in table context caused voodoo mode.
+Line: 1 Col: 63 End tag (tt) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 69 Unexpected end tag (pre) in table context caused voodoo mode.
+Line: 1 Col: 69 End tag (pre) seen too early. Expected other end tag.
+Line: 1 Col: 75 Unexpected end tag (big) in table context caused voodoo mode.
+Line: 1 Col: 75 End tag (big) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 83 Unexpected end tag (small) in table context caused voodoo mode.
+Line: 1 Col: 83 End tag (small) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 90 Unexpected end tag (font) in table context caused voodoo mode.
+Line: 1 Col: 90 End tag (font) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 99 Unexpected end tag (select) in table context caused voodoo mode.
+Line: 1 Col: 99 Unexpected end tag (select). Ignored.
+Line: 1 Col: 104 Unexpected end tag (h1) in table context caused voodoo mode.
+Line: 1 Col: 104 End tag (h1) seen too early. Expected other end tag.
+Line: 1 Col: 109 Unexpected end tag (h2) in table context caused voodoo mode.
+Line: 1 Col: 109 End tag (h2) seen too early. Expected other end tag.
+Line: 1 Col: 114 Unexpected end tag (h3) in table context caused voodoo mode.
+Line: 1 Col: 114 End tag (h3) seen too early. Expected other end tag.
+Line: 1 Col: 119 Unexpected end tag (h4) in table context caused voodoo mode.
+Line: 1 Col: 119 End tag (h4) seen too early. Expected other end tag.
+Line: 1 Col: 124 Unexpected end tag (h5) in table context caused voodoo mode.
+Line: 1 Col: 124 End tag (h5) seen too early. Expected other end tag.
+Line: 1 Col: 129 Unexpected end tag (h6) in table context caused voodoo mode.
+Line: 1 Col: 129 End tag (h6) seen too early. Expected other end tag.
+Line: 1 Col: 136 Unexpected end tag (body) in the table row phase. Ignored.
+Line: 1 Col: 141 Unexpected end tag (br) in table context caused voodoo mode.
+Line: 1 Col: 141 Unexpected end tag (br). Treated as br element.
+Line: 1 Col: 145 Unexpected end tag (a) in table context caused voodoo mode.
+Line: 1 Col: 145 End tag (a) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 151 Unexpected end tag (img) in table context caused voodoo mode.
+Line: 1 Col: 151 This element (img) has no end tag.
+Line: 1 Col: 159 Unexpected end tag (title) in table context caused voodoo mode.
+Line: 1 Col: 159 Unexpected end tag (title). Ignored.
+Line: 1 Col: 166 Unexpected end tag (span) in table context caused voodoo mode.
+Line: 1 Col: 166 Unexpected end tag (span). Ignored.
+Line: 1 Col: 174 Unexpected end tag (style) in table context caused voodoo mode.
+Line: 1 Col: 174 Unexpected end tag (style). Ignored.
+Line: 1 Col: 183 Unexpected end tag (script) in table context caused voodoo mode.
+Line: 1 Col: 183 Unexpected end tag (script). Ignored.
+Line: 1 Col: 196 Unexpected end tag (th). Ignored.
+Line: 1 Col: 201 Unexpected end tag (td). Ignored.
+Line: 1 Col: 206 Unexpected end tag (tr). Ignored.
+Line: 1 Col: 214 This element (frame) has no end tag.
+Line: 1 Col: 221 This element (area) has no end tag.
+Line: 1 Col: 228 Unexpected end tag (link). Ignored.
+Line: 1 Col: 236 This element (param) has no end tag.
+Line: 1 Col: 241 This element (hr) has no end tag.
+Line: 1 Col: 249 This element (input) has no end tag.
+Line: 1 Col: 255 Unexpected end tag (col). Ignored.
+Line: 1 Col: 262 Unexpected end tag (base). Ignored.
+Line: 1 Col: 269 Unexpected end tag (meta). Ignored.
+Line: 1 Col: 280 This element (basefont) has no end tag.
+Line: 1 Col: 290 This element (bgsound) has no end tag.
+Line: 1 Col: 298 This element (embed) has no end tag.
+Line: 1 Col: 307 This element (spacer) has no end tag.
+Line: 1 Col: 311 Unexpected end tag (p). Ignored.
+Line: 1 Col: 316 End tag (dd) seen too early. Expected other end tag.
+Line: 1 Col: 321 End tag (dt) seen too early. Expected other end tag.
+Line: 1 Col: 331 Unexpected end tag (caption). Ignored.
+Line: 1 Col: 342 Unexpected end tag (colgroup). Ignored.
+Line: 1 Col: 350 Unexpected end tag (tbody). Ignored.
+Line: 1 Col: 358 Unexpected end tag (tfoot). Ignored.
+Line: 1 Col: 366 Unexpected end tag (thead). Ignored.
+Line: 1 Col: 376 End tag (address) seen too early. Expected other end tag.
+Line: 1 Col: 389 End tag (blockquote) seen too early. Expected other end tag.
+Line: 1 Col: 398 End tag (center) seen too early. Expected other end tag.
+Line: 1 Col: 404 Unexpected end tag (dir). Ignored.
+Line: 1 Col: 410 End tag (div) seen too early. Expected other end tag.
+Line: 1 Col: 415 End tag (dl) seen too early. Expected other end tag.
+Line: 1 Col: 426 End tag (fieldset) seen too early. Expected other end tag.
+Line: 1 Col: 436 End tag (listing) seen too early. Expected other end tag.
+Line: 1 Col: 443 End tag (menu) seen too early. Expected other end tag.
+Line: 1 Col: 448 End tag (ol) seen too early. Expected other end tag.
+Line: 1 Col: 453 End tag (ul) seen too early. Expected other end tag.
+Line: 1 Col: 458 End tag (li) seen too early. Expected other end tag.
+Line: 1 Col: 465 End tag (nobr) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 471 This element (wbr) has no end tag.
+Line: 1 Col: 487 End tag (button) seen too early. Expected other end tag.
+Line: 1 Col: 497 End tag (marquee) seen too early. Expected other end tag.
+Line: 1 Col: 506 End tag (object) seen too early. Expected other end tag.
+Line: 1 Col: 524 Unexpected end tag (html). Ignored.
+Line: 1 Col: 524 Unexpected end tag (frameset). Ignored.
+Line: 1 Col: 531 Unexpected end tag (head). Ignored.
+Line: 1 Col: 540 Unexpected end tag (iframe). Ignored.
+Line: 1 Col: 548 This element (image) has no end tag.
+Line: 1 Col: 558 This element (isindex) has no end tag.
+Line: 1 Col: 568 Unexpected end tag (noembed). Ignored.
+Line: 1 Col: 579 Unexpected end tag (noframes). Ignored.
+Line: 1 Col: 590 Unexpected end tag (noscript). Ignored.
+Line: 1 Col: 601 Unexpected end tag (optgroup). Ignored.
+Line: 1 Col: 610 Unexpected end tag (option). Ignored.
+Line: 1 Col: 622 Unexpected end tag (plaintext). Ignored.
+Line: 1 Col: 633 Unexpected end tag (textarea). Ignored.
+#document
+| 
+|   
+|   
+|     

+|     
+|       
+|         
+|     

+
+#data
+
+#errors
+Line: 1 Col: 10 Unexpected start tag (frameset). Expected DOCTYPE.
+Line: 1 Col: 10 Expected closing tag. Unexpected end of file.
+#document
+| 
+|   
+|   
diff --git a/libgo/go/html/testdata/webkit/tests10.dat b/libgo/go/html/testdata/webkit/tests10.dat
new file mode 100644
index 000000000..877c9a3d7
--- /dev/null
+++ b/libgo/go/html/testdata/webkit/tests10.dat
@@ -0,0 +1,430 @@
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+
+#data
+
+#errors
+35: Stray “svg” start tag.
+42: Stray end tag “svg”
+#document
+| 
+| 
+|   
+|   
+|     
+#errors
+43: Stray “svg” start tag.
+50: Stray end tag “svg”
+#document
+| 
+| 
+|   
+|   
+|     

+#errors
+34: Start tag “svg” seen in “table”.
+41: Stray end tag “svg”.
+#document
+| 
+| 
+|   
+|   
+|     
+|     
+
+#data
+
foo

+#errors
+34: Start tag “svg” seen in “table”.
+46: Stray end tag “g”.
+53: Stray end tag “svg”.
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|         "foo"
+|     
+
+#data
+
foobar

+#errors
+34: Start tag “svg” seen in “table”.
+46: Stray end tag “g”.
+58: Stray end tag “g”.
+65: Stray end tag “svg”.
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|         "foo"
+|       
+|         "bar"
+|     
+
+#data
+
foobar

+#errors
+41: Start tag “svg” seen in “table”.
+53: Stray end tag “g”.
+65: Stray end tag “g”.
+72: Stray end tag “svg”.
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|         "foo"
+|       
+|         "bar"
+|     
+|       
+
+#data
+
foobar

+#errors
+45: Start tag “svg” seen in “table”.
+57: Stray end tag “g”.
+69: Stray end tag “g”.
+76: Stray end tag “svg”.
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|         "foo"
+|       
+|         "bar"
+|     
+|       
+|         
+
+#data
+
foobar

+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|         
+|           

+|             
+|               
+|                 "foo"
+|               
+|                 "bar"
+
+#data
+
foobar
baz

+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|         
+|           

+|             
+|               
+|                 "foo"
+|               
+|                 "bar"
+|             

+|               "baz"
+
+#data
+
foobar
baz

+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       

+|         
+|           
+|             "foo"
+|           
+|             "bar"
+|         

+|           "baz"
+
+#data
+
foobar
baz
quux
+#errors
+70: HTML start tag “p” in a foreign namespace context.
+81: “table” closed but “caption” was still open.
+#document
+| 
+| 
+|   
+|   
+|     
+|       

+|         
+|           
+|             "foo"
+|           
+|             "bar"
+|         

+|           "baz"
+|     

+|       "quux"
+
+#data
+
foobarbaz
quux
+#errors
+78: “table” closed but “caption” was still open.
+78: Unclosed elements on stack.
+#document
+| 
+| 
+|   
+|   
+|     
+|       

+|         
+|           
+|             "foo"
+|           
+|             "bar"
+|           "baz"
+|     

+|       "quux"
+
+#data
+foobar
baz
quux
+#errors
+44: Start tag “svg” seen in “table”.
+56: Stray end tag “g”.
+68: Stray end tag “g”.
+71: HTML start tag “p” in a foreign namespace context.
+71: Start tag “p” seen in “table”.
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|         "foo"
+|       
+|         "bar"
+|     

+|       "baz"
+|     
+|       +|     

+|       "quux"
+
+#data
+


quux
+#errors
+50: Stray “svg” start tag.
+54: Stray “g” start tag.
+62: Stray end tag “g”
+66: Stray “g” start tag.
+74: Stray end tag “g”
+77: Stray “p” start tag.
+88: “table” end tag with “select” open.
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|         
+|           

+|             
quux
+#errors
+36: Start tag “select” seen in “table”.
+42: Stray “svg” start tag.
+46: Stray “g” start tag.
+54: Stray end tag “g”
+58: Stray “g” start tag.
+66: Stray end tag “g”
+69: Stray “p” start tag.
+80: “table” end tag with “select” open.
+#document
+| 
+| 
+|   
+|   
+|     
+|     

+|       "quux"
+
+#data
+foobar
baz
+#errors
+41: Stray “svg” start tag.
+68: HTML start tag “p” in a foreign namespace context.
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|         "foo"
+|       
+|         "bar"
+|     

+|       "baz"
+
+#data
+foobar
baz
+#errors
+34: Stray “svg” start tag.
+61: HTML start tag “p” in a foreign namespace context.
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|         "foo"
+|       
+|         "bar"
+|     

+|       "baz"
+
+#data
+

+#errors
+31: Stray “svg” start tag.
+35: Stray “g” start tag.
+40: Stray end tag “g”
+44: Stray “g” start tag.
+49: Stray end tag “g”
+52: Stray “p” start tag.
+58: Stray “span” start tag.
+58: End of file seen and there were open elements.
+#document
+| 
+| 
+|   
+|   
+
+#data
+

+#errors
+42: Stray “svg” start tag.
+46: Stray “g” start tag.
+51: Stray end tag “g”
+55: Stray “g” start tag.
+60: Stray end tag “g”
+63: Stray “p” start tag.
+69: Stray “span” start tag.
+#document
+| 
+| 
+|   
+|   
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     xlink:href="foo"
+|     
+|       xlink href="foo"
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     xlink:href="foo"
+|     xml:lang="en"
+|     
+|       
+|         xlink href="foo"
+|         xml lang="en"
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     xlink:href="foo"
+|     xml:lang="en"
+|     
+|       
+|         xlink href="foo"
+|         xml lang="en"
+
+#data
+bar
+#errors
+#document
+| 
+| 
+|   
+|   
+|     xlink:href="foo"
+|     xml:lang="en"
+|     
+|       
+|         xlink href="foo"
+|         xml lang="en"
+|       "bar"
diff --git a/libgo/go/html/testdata/webkit/tests11.dat b/libgo/go/html/testdata/webkit/tests11.dat
new file mode 100644
index 000000000..638cde479
--- /dev/null
+++ b/libgo/go/html/testdata/webkit/tests11.dat
@@ -0,0 +1,482 @@
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       attributeName=""
+|       attributeType=""
+|       baseFrequency=""
+|       baseProfile=""
+|       calcMode=""
+|       clipPathUnits=""
+|       contentScriptType=""
+|       contentStyleType=""
+|       diffuseConstant=""
+|       edgeMode=""
+|       externalResourcesRequired=""
+|       filterRes=""
+|       filterUnits=""
+|       glyphRef=""
+|       gradientTransform=""
+|       gradientUnits=""
+|       kernelMatrix=""
+|       kernelUnitLength=""
+|       keyPoints=""
+|       keySplines=""
+|       keyTimes=""
+|       lengthAdjust=""
+|       limitingConeAngle=""
+|       markerHeight=""
+|       markerUnits=""
+|       markerWidth=""
+|       maskContentUnits=""
+|       maskUnits=""
+|       numOctaves=""
+|       pathLength=""
+|       patternContentUnits=""
+|       patternTransform=""
+|       patternUnits=""
+|       pointsAtX=""
+|       pointsAtY=""
+|       pointsAtZ=""
+|       preserveAlpha=""
+|       preserveAspectRatio=""
+|       primitiveUnits=""
+|       refX=""
+|       refY=""
+|       repeatCount=""
+|       repeatDur=""
+|       requiredExtensions=""
+|       requiredFeatures=""
+|       specularConstant=""
+|       specularExponent=""
+|       spreadMethod=""
+|       startOffset=""
+|       stdDeviation=""
+|       stitchTiles=""
+|       surfaceScale=""
+|       systemLanguage=""
+|       tableValues=""
+|       targetX=""
+|       targetY=""
+|       textLength=""
+|       viewBox=""
+|       viewTarget=""
+|       xChannelSelector=""
+|       yChannelSelector=""
+|       zoomAndPan=""
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       attributeName=""
+|       attributeType=""
+|       baseFrequency=""
+|       baseProfile=""
+|       calcMode=""
+|       clipPathUnits=""
+|       contentScriptType=""
+|       contentStyleType=""
+|       diffuseConstant=""
+|       edgeMode=""
+|       externalResourcesRequired=""
+|       filterRes=""
+|       filterUnits=""
+|       glyphRef=""
+|       gradientTransform=""
+|       gradientUnits=""
+|       kernelMatrix=""
+|       kernelUnitLength=""
+|       keyPoints=""
+|       keySplines=""
+|       keyTimes=""
+|       lengthAdjust=""
+|       limitingConeAngle=""
+|       markerHeight=""
+|       markerUnits=""
+|       markerWidth=""
+|       maskContentUnits=""
+|       maskUnits=""
+|       numOctaves=""
+|       pathLength=""
+|       patternContentUnits=""
+|       patternTransform=""
+|       patternUnits=""
+|       pointsAtX=""
+|       pointsAtY=""
+|       pointsAtZ=""
+|       preserveAlpha=""
+|       preserveAspectRatio=""
+|       primitiveUnits=""
+|       refX=""
+|       refY=""
+|       repeatCount=""
+|       repeatDur=""
+|       requiredExtensions=""
+|       requiredFeatures=""
+|       specularConstant=""
+|       specularExponent=""
+|       spreadMethod=""
+|       startOffset=""
+|       stdDeviation=""
+|       stitchTiles=""
+|       surfaceScale=""
+|       systemLanguage=""
+|       tableValues=""
+|       targetX=""
+|       targetY=""
+|       textLength=""
+|       viewBox=""
+|       viewTarget=""
+|       xChannelSelector=""
+|       yChannelSelector=""
+|       zoomAndPan=""
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       attributeName=""
+|       attributeType=""
+|       baseFrequency=""
+|       baseProfile=""
+|       calcMode=""
+|       clipPathUnits=""
+|       contentScriptType=""
+|       contentStyleType=""
+|       diffuseConstant=""
+|       edgeMode=""
+|       externalResourcesRequired=""
+|       filterRes=""
+|       filterUnits=""
+|       glyphRef=""
+|       gradientTransform=""
+|       gradientUnits=""
+|       kernelMatrix=""
+|       kernelUnitLength=""
+|       keyPoints=""
+|       keySplines=""
+|       keyTimes=""
+|       lengthAdjust=""
+|       limitingConeAngle=""
+|       markerHeight=""
+|       markerUnits=""
+|       markerWidth=""
+|       maskContentUnits=""
+|       maskUnits=""
+|       numOctaves=""
+|       pathLength=""
+|       patternContentUnits=""
+|       patternTransform=""
+|       patternUnits=""
+|       pointsAtX=""
+|       pointsAtY=""
+|       pointsAtZ=""
+|       preserveAlpha=""
+|       preserveAspectRatio=""
+|       primitiveUnits=""
+|       refX=""
+|       refY=""
+|       repeatCount=""
+|       repeatDur=""
+|       requiredExtensions=""
+|       requiredFeatures=""
+|       specularConstant=""
+|       specularExponent=""
+|       spreadMethod=""
+|       startOffset=""
+|       stdDeviation=""
+|       stitchTiles=""
+|       surfaceScale=""
+|       systemLanguage=""
+|       tableValues=""
+|       targetX=""
+|       targetY=""
+|       textLength=""
+|       viewBox=""
+|       viewTarget=""
+|       xChannelSelector=""
+|       yChannelSelector=""
+|       zoomAndPan=""
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       attributename=""
+|       attributetype=""
+|       basefrequency=""
+|       baseprofile=""
+|       calcmode=""
+|       clippathunits=""
+|       contentscripttype=""
+|       contentstyletype=""
+|       diffuseconstant=""
+|       edgemode=""
+|       externalresourcesrequired=""
+|       filterres=""
+|       filterunits=""
+|       glyphref=""
+|       gradienttransform=""
+|       gradientunits=""
+|       kernelmatrix=""
+|       kernelunitlength=""
+|       keypoints=""
+|       keysplines=""
+|       keytimes=""
+|       lengthadjust=""
+|       limitingconeangle=""
+|       markerheight=""
+|       markerunits=""
+|       markerwidth=""
+|       maskcontentunits=""
+|       maskunits=""
+|       numoctaves=""
+|       pathlength=""
+|       patterncontentunits=""
+|       patterntransform=""
+|       patternunits=""
+|       pointsatx=""
+|       pointsaty=""
+|       pointsatz=""
+|       preservealpha=""
+|       preserveaspectratio=""
+|       primitiveunits=""
+|       refx=""
+|       refy=""
+|       repeatcount=""
+|       repeatdur=""
+|       requiredextensions=""
+|       requiredfeatures=""
+|       specularconstant=""
+|       specularexponent=""
+|       spreadmethod=""
+|       startoffset=""
+|       stddeviation=""
+|       stitchtiles=""
+|       surfacescale=""
+|       systemlanguage=""
+|       tablevalues=""
+|       targetx=""
+|       targety=""
+|       textlength=""
+|       viewbox=""
+|       viewtarget=""
+|       xchannelselector=""
+|       ychannelselector=""
+|       zoomandpan=""
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       
diff --git a/libgo/go/html/testdata/webkit/tests12.dat b/libgo/go/html/testdata/webkit/tests12.dat
new file mode 100644
index 000000000..63107d277
--- /dev/null
+++ b/libgo/go/html/testdata/webkit/tests12.dat
@@ -0,0 +1,62 @@
+#data
+
foobazeggs
spam
quuxbar
+#errors
+#document
+| 
+| 
+|   
+|   
+|     

+|       "foo"
+|       
+|         
+|           
+|             "baz"
+|         
+|           
+|             
+|               
+|                 "eggs"
+|             
+|               
+|                 

+|                   "spam"
+|                 
+|                   
+|                     
+|                       

+|                         
+|             
+|               "quux"
+|       "bar"
+
+#data
+foobazeggs
spam
quuxbar
+#errors
+#document
+| 
+| 
+|   
+|   
+|     "foo"
+|     
+|       
+|         
+|           "baz"
+|       
+|         
+|           
+|             
+|               "eggs"
+|           
+|             
+|               

+|                 "spam"
+|               
+|                 
+|                   
+|                     

+|                       
+|           
+|             "quux"
+|     "bar"
diff --git a/libgo/go/html/testdata/webkit/tests13.dat b/libgo/go/html/testdata/webkit/tests13.dat
new file mode 100644
index 000000000..d180e8e90
--- /dev/null
+++ b/libgo/go/html/testdata/webkit/tests13.dat
@@ -0,0 +1,9 @@
+
+
+404 Not Found
+
+
Not Found

+
The requested URL /html5lib-tests/data/tests13.dat was not found on this server.

+
Additionally, a 404 Not Found
+error was encountered while trying to use an ErrorDocument to handle the request.

+
diff --git a/libgo/go/html/testdata/webkit/tests14.dat b/libgo/go/html/testdata/webkit/tests14.dat
new file mode 100644
index 000000000..72f8015f6
--- /dev/null
+++ b/libgo/go/html/testdata/webkit/tests14.dat
@@ -0,0 +1,74 @@
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|     
+
+#data
+
+#errors
+15: Unexpected start tag html
+#document
+| 
+| 
+|   abc:def="gh"
+|   
+|   
+|     
+
+#data
+
+#errors
+15: Unexpected start tag html
+#document
+| 
+| 
+|   xml:lang="bar"
+|   
+|   
+
+#data
+
+#errors
+#document
+| 
+| 
+|   123="456"
+|   
+|   
+
+#data
+
+#errors
+#document
+| 
+| 
+|   123="456"
+|   789="012"
+|   
+|   
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     789="012"
\ No newline at end of file
diff --git a/libgo/go/html/testdata/webkit/tests15.dat b/libgo/go/html/testdata/webkit/tests15.dat
new file mode 100644
index 000000000..7f016cae3
--- /dev/null
+++ b/libgo/go/html/testdata/webkit/tests15.dat
@@ -0,0 +1,208 @@
+#data
+
 
X
+#errors
+Line: 1 Col: 31 Unexpected end tag (p). Ignored.
+Line: 1 Col: 36 Expected closing tag. Unexpected end of file.
+#document
+| 
+| 
+|   
+|   
+|     

+|       
+|         
+|           
+|     
+|       
+|         
+|           " "
+|           

+|             "X"
+
+#data
+

+
X
+#errors
+Line: 1 Col: 3 Unexpected start tag (p). Expected DOCTYPE.
+Line: 1 Col: 16 Unexpected end tag (p). Ignored.
+Line: 2 Col: 4 Expected closing tag. Unexpected end of file.
+#document
+| 
+|   
+|   
+|     

+|       
+|         
+|           
+|     
+|       
+|         
+|           "
+"
+|           

+|             "X"
+
+#data
+ 
+#errors
+Line: 1 Col: 22 Unexpected end tag (html) after the (implied) root element.
+#document
+| 
+| 
+|   
+|   
+|     " "
+
+#data
+
+#errors
+Line: 1 Col: 22 Unexpected end tag (body) after the (implied) root element.
+#document
+| 
+| 
+|   
+|   
+|     
+
+#data
+
+#errors
+Line: 1 Col: 6 Unexpected start tag (html). Expected DOCTYPE.
+Line: 1 Col: 13 Unexpected end tag (html) after the (implied) root element.
+#document
+| 
+|   
+|   
+| 
+
+#data
+X
+#errors
+Line: 1 Col: 22 Unexpected end tag (body) after the (implied) root element.
+#document
+| 
+| 
+|   
+|   
+|     
+|       "X"
+
+#data
+<!doctype html><table> X<meta></table>
+#errors
+Line: 1 Col: 24 Unexpected non-space characters in table context caused voodoo mode.
+Line: 1 Col: 30 Unexpected start tag (meta) in table context caused voodoo mode.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     " X"
+|     <meta>
+|     <table>
+
+#data
+<!doctype html><table> x</table>
+#errors
+Line: 1 Col: 24 Unexpected non-space characters in table context caused voodoo mode.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     " x"
+|     <table>
+
+#data
+<!doctype html><table> x </table>
+#errors
+Line: 1 Col: 25 Unexpected non-space characters in table context caused voodoo mode.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     " x "
+|     <table>
+
+#data
+<!doctype html><table><tr> x</table>
+#errors
+Line: 1 Col: 28 Unexpected non-space characters in table context caused voodoo mode.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     " x"
+|     <table>
+|       <tbody>
+|         <tr>
+
+#data
+<!doctype html><table>X<style> <tr>x </style> </table>
+#errors
+Line: 1 Col: 23 Unexpected non-space characters in table context caused voodoo mode.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     "X"
+|     <table>
+|       <style>
+|         " <tr>x "
+|       " "
+
+#data
+<!doctype html><div><table><a>foo</a> <tr><td>bar</td> </tr></table></div>
+#errors
+Line: 1 Col: 30 Unexpected start tag (a) in table context caused voodoo mode.
+Line: 1 Col: 37 Unexpected end tag (a) in table context caused voodoo mode.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <div>
+|       <a>
+|         "foo"
+|       <table>
+|         " "
+|         <tbody>
+|           <tr>
+|             <td>
+|               "bar"
+|             " "
+
+#data
+<frame></frame></frame><frameset><frame><frameset><frame></frameset><noframes></frameset><noframes>
+#errors
+6: Start tag seen without seeing a doctype first. Expected “<!DOCTYPE html>”.
+13: Stray start tag “frame”.
+21: Stray end tag “frame”.
+29: Stray end tag “frame”.
+39: “frameset” start tag after “body” already open.
+105: End of file seen inside an [R]CDATA element.
+105: End of file seen and there were open elements.
+XXX: These errors are wrong, please fix me!
+#document
+| <html>
+|   <head>
+|   <frameset>
+|     <frame>
+|     <frameset>
+|       <frame>
+|     <noframes>
+|       "</frameset><noframes>"
+
+#data
+<!DOCTYPE html><object></html>
+#errors
+1: Expected closing tag. Unexpected end of file
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <object>
\ No newline at end of file
diff --git a/libgo/go/html/testdata/webkit/tests16.dat b/libgo/go/html/testdata/webkit/tests16.dat
new file mode 100644
index 000000000..937dba9f4
--- /dev/null
+++ b/libgo/go/html/testdata/webkit/tests16.dat
@@ -0,0 +1,2277 @@
+#data
+<!doctype html><script>
+#errors
+Line: 1 Col: 23 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|   <body>
+
+#data
+<!doctype html><script>a
+#errors
+Line: 1 Col: 24 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "a"
+|   <body>
+
+#data
+<!doctype html><script><
+#errors
+Line: 1 Col: 24 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<"
+|   <body>
+
+#data
+<!doctype html><script></
+#errors
+Line: 1 Col: 25 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</"
+|   <body>
+
+#data
+<!doctype html><script></S
+#errors
+Line: 1 Col: 26 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</S"
+|   <body>
+
+#data
+<!doctype html><script></SC
+#errors
+Line: 1 Col: 27 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</SC"
+|   <body>
+
+#data
+<!doctype html><script></SCR
+#errors
+Line: 1 Col: 28 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</SCR"
+|   <body>
+
+#data
+<!doctype html><script></SCRI
+#errors
+Line: 1 Col: 29 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</SCRI"
+|   <body>
+
+#data
+<!doctype html><script></SCRIP
+#errors
+Line: 1 Col: 30 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</SCRIP"
+|   <body>
+
+#data
+<!doctype html><script></SCRIPT
+#errors
+Line: 1 Col: 31 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</SCRIPT"
+|   <body>
+
+#data
+<!doctype html><script></SCRIPT 
+#errors
+Line: 1 Col: 32 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|   <body>
+
+#data
+<!doctype html><script></s
+#errors
+Line: 1 Col: 26 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</s"
+|   <body>
+
+#data
+<!doctype html><script></sc
+#errors
+Line: 1 Col: 27 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</sc"
+|   <body>
+
+#data
+<!doctype html><script></scr
+#errors
+Line: 1 Col: 28 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</scr"
+|   <body>
+
+#data
+<!doctype html><script></scri
+#errors
+Line: 1 Col: 29 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</scri"
+|   <body>
+
+#data
+<!doctype html><script></scrip
+#errors
+Line: 1 Col: 30 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</scrip"
+|   <body>
+
+#data
+<!doctype html><script></script
+#errors
+Line: 1 Col: 31 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</script"
+|   <body>
+
+#data
+<!doctype html><script></script 
+#errors
+Line: 1 Col: 32 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|   <body>
+
+#data
+<!doctype html><script><!
+#errors
+Line: 1 Col: 25 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!"
+|   <body>
+
+#data
+<!doctype html><script><!a
+#errors
+Line: 1 Col: 26 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!a"
+|   <body>
+
+#data
+<!doctype html><script><!-
+#errors
+Line: 1 Col: 26 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!-"
+|   <body>
+
+#data
+<!doctype html><script><!-a
+#errors
+Line: 1 Col: 27 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!-a"
+|   <body>
+
+#data
+<!doctype html><script><!--
+#errors
+Line: 1 Col: 27 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--"
+|   <body>
+
+#data
+<!doctype html><script><!--a
+#errors
+Line: 1 Col: 28 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--a"
+|   <body>
+
+#data
+<!doctype html><script><!--<
+#errors
+Line: 1 Col: 28 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<"
+|   <body>
+
+#data
+<!doctype html><script><!--<a
+#errors
+Line: 1 Col: 29 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<a"
+|   <body>
+
+#data
+<!doctype html><script><!--</
+#errors
+Line: 1 Col: 27 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--</"
+|   <body>
+
+#data
+<!doctype html><script><!--</script
+#errors
+Line: 1 Col: 35 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--</script"
+|   <body>
+
+#data
+<!doctype html><script><!--</script 
+#errors
+Line: 1 Col: 36 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--"
+|   <body>
+
+#data
+<!doctype html><script><!--<s
+#errors
+Line: 1 Col: 29 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<s"
+|   <body>
+
+#data
+<!doctype html><script><!--<script
+#errors
+Line: 1 Col: 34 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script"
+|   <body>
+
+#data
+<!doctype html><script><!--<script 
+#errors
+Line: 1 Col: 35 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script "
+|   <body>
+
+#data
+<!doctype html><script><!--<script <
+#errors
+Line: 1 Col: 36 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script <"
+|   <body>
+
+#data
+<!doctype html><script><!--<script <a
+#errors
+Line: 1 Col: 37 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script <a"
+|   <body>
+
+#data
+<!doctype html><script><!--<script </
+#errors
+Line: 1 Col: 37 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </"
+|   <body>
+
+#data
+<!doctype html><script><!--<script </s
+#errors
+Line: 1 Col: 38 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </s"
+|   <body>
+
+#data
+<!doctype html><script><!--<script </script
+#errors
+Line: 1 Col: 43 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script"
+|   <body>
+
+#data
+<!doctype html><script><!--<script </scripta
+#errors
+Line: 1 Col: 44 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </scripta"
+|   <body>
+
+#data
+<!doctype html><script><!--<script </script 
+#errors
+Line: 1 Col: 44 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script "
+|   <body>
+
+#data
+<!doctype html><script><!--<script </script>
+#errors
+Line: 1 Col: 44 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script>"
+|   <body>
+
+#data
+<!doctype html><script><!--<script </script/
+#errors
+Line: 1 Col: 44 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script/"
+|   <body>
+
+#data
+<!doctype html><script><!--<script </script <
+#errors
+Line: 1 Col: 45 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script <"
+|   <body>
+
+#data
+<!doctype html><script><!--<script </script <a
+#errors
+Line: 1 Col: 46 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script <a"
+|   <body>
+
+#data
+<!doctype html><script><!--<script </script </
+#errors
+Line: 1 Col: 46 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script </"
+|   <body>
+
+#data
+<!doctype html><script><!--<script </script </script
+#errors
+Line: 1 Col: 52 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script </script"
+|   <body>
+
+#data
+<!doctype html><script><!--<script </script </script 
+#errors
+Line: 1 Col: 53 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script "
+|   <body>
+
+#data
+<!doctype html><script><!--<script </script </script/
+#errors
+Line: 1 Col: 53 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script "
+|   <body>
+
+#data
+<!doctype html><script><!--<script </script </script>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script "
+|   <body>
+
+#data
+<!doctype html><script><!--<script -
+#errors
+Line: 1 Col: 36 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -"
+|   <body>
+
+#data
+<!doctype html><script><!--<script -a
+#errors
+Line: 1 Col: 37 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -a"
+|   <body>
+
+#data
+<!doctype html><script><!--<script -<
+#errors
+Line: 1 Col: 37 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -<"
+|   <body>
+
+#data
+<!doctype html><script><!--<script --
+#errors
+Line: 1 Col: 37 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script --"
+|   <body>
+
+#data
+<!doctype html><script><!--<script --a
+#errors
+Line: 1 Col: 38 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script --a"
+|   <body>
+
+#data
+<!doctype html><script><!--<script --<
+#errors
+Line: 1 Col: 38 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script --<"
+|   <body>
+
+#data
+<!doctype html><script><!--<script -->
+#errors
+Line: 1 Col: 38 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -->"
+|   <body>
+
+#data
+<!doctype html><script><!--<script --><
+#errors
+Line: 1 Col: 39 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script --><"
+|   <body>
+
+#data
+<!doctype html><script><!--<script --></
+#errors
+Line: 1 Col: 40 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script --></"
+|   <body>
+
+#data
+<!doctype html><script><!--<script --></script
+#errors
+Line: 1 Col: 46 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script --></script"
+|   <body>
+
+#data
+<!doctype html><script><!--<script --></script 
+#errors
+Line: 1 Col: 47 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -->"
+|   <body>
+
+#data
+<!doctype html><script><!--<script --></script/
+#errors
+Line: 1 Col: 47 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -->"
+|   <body>
+
+#data
+<!doctype html><script><!--<script --></script>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -->"
+|   <body>
+
+#data
+<!doctype html><script><!--<script><\/script>--></script>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script><\/script>-->"
+|   <body>
+
+#data
+<!doctype html><script><!--<script></scr'+'ipt>--></script>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></scr'+'ipt>-->"
+|   <body>
+
+#data
+<!doctype html><script><!--<script></script><script></script></script>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></script><script></script>"
+|   <body>
+
+#data
+<!doctype html><script><!--<script></script><script></script>--><!--</script>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></script><script></script>--><!--"
+|   <body>
+
+#data
+<!doctype html><script><!--<script></script><script></script>-- ></script>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></script><script></script>-- >"
+|   <body>
+
+#data
+<!doctype html><script><!--<script></script><script></script>- -></script>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></script><script></script>- ->"
+|   <body>
+
+#data
+<!doctype html><script><!--<script></script><script></script>- - ></script>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></script><script></script>- - >"
+|   <body>
+
+#data
+<!doctype html><script><!--<script></script><script></script>-></script>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></script><script></script>->"
+|   <body>
+
+#data
+<!doctype html><script><!--<script>--!></script>X
+#errors
+Line: 1 Col: 49 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script>--!></script>X"
+|   <body>
+
+#data
+<!doctype html><script><!--<scr'+'ipt></script>--></script>
+#errors
+Line: 1 Col: 59 Unexpected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<scr'+'ipt>"
+|   <body>
+|     "-->"
+
+#data
+<!doctype html><script><!--<script></scr'+'ipt></script>X
+#errors
+Line: 1 Col: 57 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></scr'+'ipt></script>X"
+|   <body>
+
+#data
+<!doctype html><style><!--<style></style>--></style>
+#errors
+Line: 1 Col: 52 Unexpected end tag (style).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <style>
+|       "<!--<style>"
+|   <body>
+|     "-->"
+
+#data
+<!doctype html><style><!--</style>X
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <style>
+|       "<!--"
+|   <body>
+|     "X"
+
+#data
+<!doctype html><style><!--...</style>...--></style>
+#errors
+Line: 1 Col: 51 Unexpected end tag (style).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <style>
+|       "<!--..."
+|   <body>
+|     "...-->"
+
+#data
+<!doctype html><style><!--<br><html xmlns:v="urn:schemas-microsoft-com:vml"><!--[if !mso]><style></style>X
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <style>
+|       "<!--<br><html xmlns:v="urn:schemas-microsoft-com:vml"><!--[if !mso]><style>"
+|   <body>
+|     "X"
+
+#data
+<!doctype html><style><!--...<style><!--...--!></style>--></style>
+#errors
+Line: 1 Col: 66 Unexpected end tag (style).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <style>
+|       "<!--...<style><!--...--!>"
+|   <body>
+|     "-->"
+
+#data
+<!doctype html><style><!--...</style><!-- --><style>@import ...</style>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <style>
+|       "<!--..."
+|     <!--   -->
+|     <style>
+|       "@import ..."
+|   <body>
+
+#data
+<!doctype html><style>...<style><!--...</style><!-- --></style>
+#errors
+Line: 1 Col: 63 Unexpected end tag (style).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <style>
+|       "...<style><!--..."
+|     <!--   -->
+|   <body>
+
+#data
+<!doctype html><style>...<!--[if IE]><style>...</style>X
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <style>
+|       "...<!--[if IE]><style>..."
+|   <body>
+|     "X"
+
+#data
+<!doctype html><title><!--<title>-->
+#errors
+Line: 1 Col: 52 Unexpected end tag (title).
+#document
+| 
+| 
+|   
+|     
+|       "<!--<title>"
+|   <body>
+|     "-->"
+
+#data
+<!doctype html><title></title>
+#errors
+#document
+| 
+| 
+|   
+|     
+|       ""
+|   
+
+#data
+foo/title><link></head><body>X
+#errors
+Line: 1 Col: 52 Unexpected end of file. Expected end tag (title).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <title>
+|       "foo/title><link></head><body>X"
+|   <body>
+
+#data
+<!doctype html><noscript><!--<noscript></noscript>--></noscript>
+#errors
+Line: 1 Col: 64 Unexpected end tag (noscript).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <noscript>
+|       "<!--<noscript>"
+|   <body>
+|     "-->"
+
+#data
+<!doctype html><noscript><!--</noscript>X<noscript>--></noscript>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <noscript>
+|       "<!--"
+|   <body>
+|     "X"
+|     <noscript>
+|       "-->"
+
+#data
+<!doctype html><noscript><iframe></noscript>X
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <noscript>
+|       "<iframe>"
+|   <body>
+|     "X"
+
+#data
+<!doctype html><noframes><!--<noframes></noframes>--></noframes>
+#errors
+Line: 1 Col: 64 Unexpected end tag (noframes).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <noframes>
+|       "<!--<noframes>"
+|   <body>
+|     "-->"
+
+#data
+<!doctype html><noframes><body><script><!--...</script></body></noframes></html>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <noframes>
+|       "<body><script><!--...</script></body>"
+|   <body>
+
+#data
+<!doctype html><textarea><!--<textarea></textarea>--></textarea>
+#errors
+Line: 1 Col: 64 Unexpected end tag (textarea).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <textarea>
+|       "<!--<textarea>"
+|     "-->"
+
+#data
+<!doctype html><textarea></textarea></textarea>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <textarea>
+|       "</textarea>"
+
+#data
+<!doctype html><iframe><!--<iframe></iframe>--></iframe>
+#errors
+Line: 1 Col: 56 Unexpected end tag (iframe).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <iframe>
+|       "<!--<iframe>"
+|     "-->"
+
+#data
+<!doctype html><iframe>...<!--X->...<!--/X->...</iframe>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <iframe>
+|       "...<!--X->...<!--/X->..."
+
+#data
+<!doctype html><xmp><!--<xmp></xmp>--></xmp>
+#errors
+Line: 1 Col: 44 Unexpected end tag (xmp).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <xmp>
+|       "<!--<xmp>"
+|     "-->"
+
+#data
+<!doctype html><noembed><!--<noembed></noembed>--></noembed>
+#errors
+Line: 1 Col: 60 Unexpected end tag (noembed).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <noembed>
+|       "<!--<noembed>"
+|     "-->"
+
+#data
+<script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 8 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|   <body>
+
+#data
+<script>a
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 9 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "a"
+|   <body>
+
+#data
+<script><
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 9 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<"
+|   <body>
+
+#data
+<script></
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 10 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</"
+|   <body>
+
+#data
+<script></S
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 11 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</S"
+|   <body>
+
+#data
+<script></SC
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 12 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</SC"
+|   <body>
+
+#data
+<script></SCR
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 13 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</SCR"
+|   <body>
+
+#data
+<script></SCRI
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 14 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</SCRI"
+|   <body>
+
+#data
+<script></SCRIP
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 15 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</SCRIP"
+|   <body>
+
+#data
+<script></SCRIPT
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 16 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</SCRIPT"
+|   <body>
+
+#data
+<script></SCRIPT 
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 17 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|   <body>
+
+#data
+<script></s
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 11 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</s"
+|   <body>
+
+#data
+<script></sc
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 12 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</sc"
+|   <body>
+
+#data
+<script></scr
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 13 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</scr"
+|   <body>
+
+#data
+<script></scri
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 14 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</scri"
+|   <body>
+
+#data
+<script></scrip
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 15 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</scrip"
+|   <body>
+
+#data
+<script></script
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 16 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</script"
+|   <body>
+
+#data
+<script></script 
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 17 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|   <body>
+
+#data
+<script><!
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 10 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!"
+|   <body>
+
+#data
+<script><!a
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 11 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!a"
+|   <body>
+
+#data
+<script><!-
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 11 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!-"
+|   <body>
+
+#data
+<script><!-a
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 12 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!-a"
+|   <body>
+
+#data
+<script><!--
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 12 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--"
+|   <body>
+
+#data
+<script><!--a
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 13 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--a"
+|   <body>
+
+#data
+<script><!--<
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 13 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<"
+|   <body>
+
+#data
+<script><!--<a
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 14 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<a"
+|   <body>
+
+#data
+<script><!--</
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 14 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--</"
+|   <body>
+
+#data
+<script><!--</script
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 20 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--</script"
+|   <body>
+
+#data
+<script><!--</script 
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 21 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--"
+|   <body>
+
+#data
+<script><!--<s
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 14 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<s"
+|   <body>
+
+#data
+<script><!--<script
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 19 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script"
+|   <body>
+
+#data
+<script><!--<script 
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 20 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script "
+|   <body>
+
+#data
+<script><!--<script <
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 21 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script <"
+|   <body>
+
+#data
+<script><!--<script <a
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 22 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script <a"
+|   <body>
+
+#data
+<script><!--<script </
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 22 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </"
+|   <body>
+
+#data
+<script><!--<script </s
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 23 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </s"
+|   <body>
+
+#data
+<script><!--<script </script
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 28 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script"
+|   <body>
+
+#data
+<script><!--<script </scripta
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 29 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </scripta"
+|   <body>
+
+#data
+<script><!--<script </script 
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 29 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script "
+|   <body>
+
+#data
+<script><!--<script </script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 29 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script>"
+|   <body>
+
+#data
+<script><!--<script </script/
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 29 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script/"
+|   <body>
+
+#data
+<script><!--<script </script <
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 30 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script <"
+|   <body>
+
+#data
+<script><!--<script </script <a
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 31 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script <a"
+|   <body>
+
+#data
+<script><!--<script </script </
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 31 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script </"
+|   <body>
+
+#data
+<script><!--<script </script </script
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 38 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script </script"
+|   <body>
+
+#data
+<script><!--<script </script </script 
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 38 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script "
+|   <body>
+
+#data
+<script><!--<script </script </script/
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 38 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script "
+|   <body>
+
+#data
+<script><!--<script </script </script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script "
+|   <body>
+
+#data
+<script><!--<script -
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 21 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -"
+|   <body>
+
+#data
+<script><!--<script -a
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 22 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -a"
+|   <body>
+
+#data
+<script><!--<script --
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 22 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script --"
+|   <body>
+
+#data
+<script><!--<script --a
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 23 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script --a"
+|   <body>
+
+#data
+<script><!--<script -->
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 23 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -->"
+|   <body>
+
+#data
+<script><!--<script --><
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 24 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script --><"
+|   <body>
+
+#data
+<script><!--<script --></
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 25 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script --></"
+|   <body>
+
+#data
+<script><!--<script --></script
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 31 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script --></script"
+|   <body>
+
+#data
+<script><!--<script --></script 
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 32 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -->"
+|   <body>
+
+#data
+<script><!--<script --></script/
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 32 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -->"
+|   <body>
+
+#data
+<script><!--<script --></script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -->"
+|   <body>
+
+#data
+<script><!--<script><\/script>--></script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script><\/script>-->"
+|   <body>
+
+#data
+<script><!--<script></scr'+'ipt>--></script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></scr'+'ipt>-->"
+|   <body>
+
+#data
+<script><!--<script></script><script></script></script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></script><script></script>"
+|   <body>
+
+#data
+<script><!--<script></script><script></script>--><!--</script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></script><script></script>--><!--"
+|   <body>
+
+#data
+<script><!--<script></script><script></script>-- ></script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></script><script></script>-- >"
+|   <body>
+
+#data
+<script><!--<script></script><script></script>- -></script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></script><script></script>- ->"
+|   <body>
+
+#data
+<script><!--<script></script><script></script>- - ></script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></script><script></script>- - >"
+|   <body>
+
+#data
+<script><!--<script></script><script></script>-></script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></script><script></script>->"
+|   <body>
+
+#data
+<script><!--<script>--!></script>X
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 34 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script>--!></script>X"
+|   <body>
+
+#data
+<script><!--<scr'+'ipt></script>--></script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 44 Unexpected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<scr'+'ipt>"
+|   <body>
+|     "-->"
+
+#data
+<script><!--<script></scr'+'ipt></script>X
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 42 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></scr'+'ipt></script>X"
+|   <body>
+
+#data
+<style><!--<style></style>--></style>
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+Line: 1 Col: 37 Unexpected end tag (style).
+#document
+| <html>
+|   <head>
+|     <style>
+|       "<!--<style>"
+|   <body>
+|     "-->"
+
+#data
+<style><!--</style>X
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <style>
+|       "<!--"
+|   <body>
+|     "X"
+
+#data
+<style><!--...</style>...--></style>
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+Line: 1 Col: 36 Unexpected end tag (style).
+#document
+| <html>
+|   <head>
+|     <style>
+|       "<!--..."
+|   <body>
+|     "...-->"
+
+#data
+<style><!--<br><html xmlns:v="urn:schemas-microsoft-com:vml"><!--[if !mso]><style></style>X
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <style>
+|       "<!--<br><html xmlns:v="urn:schemas-microsoft-com:vml"><!--[if !mso]><style>"
+|   <body>
+|     "X"
+
+#data
+<style><!--...<style><!--...--!></style>--></style>
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+Line: 1 Col: 51 Unexpected end tag (style).
+#document
+| <html>
+|   <head>
+|     <style>
+|       "<!--...<style><!--...--!>"
+|   <body>
+|     "-->"
+
+#data
+<style><!--...</style><!-- --><style>@import ...</style>
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <style>
+|       "<!--..."
+|     <!--   -->
+|     <style>
+|       "@import ..."
+|   <body>
+
+#data
+<style>...<style><!--...</style><!-- --></style>
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+Line: 1 Col: 48 Unexpected end tag (style).
+#document
+| <html>
+|   <head>
+|     <style>
+|       "...<style><!--..."
+|     <!--   -->
+|   <body>
+
+#data
+<style>...<!--[if IE]><style>...</style>X
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <style>
+|       "...<!--[if IE]><style>..."
+|   <body>
+|     "X"
+
+#data
+<title><!--<title>-->
+#errors
+Line: 1 Col: 7 Unexpected start tag (title). Expected DOCTYPE.
+Line: 1 Col: 37 Unexpected end tag (title).
+#document
+| 
+|   
+|     
+|       "<!--<title>"
+|   <body>
+|     "-->"
+
+#data
+<title></title>
+#errors
+Line: 1 Col: 7 Unexpected start tag (title). Expected DOCTYPE.
+#document
+| 
+|   
+|     
+|       ""
+|   
+
+#data
+foo/title><link></head><body>X
+#errors
+Line: 1 Col: 7 Unexpected start tag (title). Expected DOCTYPE.
+Line: 1 Col: 37 Unexpected end of file. Expected end tag (title).
+#document
+| <html>
+|   <head>
+|     <title>
+|       "foo/title><link></head><body>X"
+|   <body>
+
+#data
+<noscript><!--<noscript></noscript>--></noscript>
+#errors
+Line: 1 Col: 10 Unexpected start tag (noscript). Expected DOCTYPE.
+Line: 1 Col: 49 Unexpected end tag (noscript).
+#document
+| <html>
+|   <head>
+|     <noscript>
+|       "<!--<noscript>"
+|   <body>
+|     "-->"
+
+#data
+<noscript><!--</noscript>X<noscript>--></noscript>
+#errors
+Line: 1 Col: 10 Unexpected start tag (noscript). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <noscript>
+|       "<!--"
+|   <body>
+|     "X"
+|     <noscript>
+|       "-->"
+
+#data
+<noscript><iframe></noscript>X
+#errors
+Line: 1 Col: 10 Unexpected start tag (noscript). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <noscript>
+|       "<iframe>"
+|   <body>
+|     "X"
+
+#data
+<noframes><!--<noframes></noframes>--></noframes>
+#errors
+Line: 1 Col: 10 Unexpected start tag (noframes). Expected DOCTYPE.
+Line: 1 Col: 49 Unexpected end tag (noframes).
+#document
+| <html>
+|   <head>
+|     <noframes>
+|       "<!--<noframes>"
+|   <body>
+|     "-->"
+
+#data
+<noframes><body><script><!--...</script></body></noframes></html>
+#errors
+Line: 1 Col: 10 Unexpected start tag (noframes). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <noframes>
+|       "<body><script><!--...</script></body>"
+|   <body>
+
+#data
+<textarea><!--<textarea></textarea>--></textarea>
+#errors
+Line: 1 Col: 10 Unexpected start tag (textarea). Expected DOCTYPE.
+Line: 1 Col: 49 Unexpected end tag (textarea).
+#document
+| <html>
+|   <head>
+|   <body>
+|     <textarea>
+|       "<!--<textarea>"
+|     "-->"
+
+#data
+<textarea></textarea></textarea>
+#errors
+Line: 1 Col: 10 Unexpected start tag (textarea). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <body>
+|     <textarea>
+|       "</textarea>"
+
+#data
+<iframe><!--<iframe></iframe>--></iframe>
+#errors
+Line: 1 Col: 8 Unexpected start tag (iframe). Expected DOCTYPE.
+Line: 1 Col: 41 Unexpected end tag (iframe).
+#document
+| <html>
+|   <head>
+|   <body>
+|     <iframe>
+|       "<!--<iframe>"
+|     "-->"
+
+#data
+<iframe>...<!--X->...<!--/X->...</iframe>
+#errors
+Line: 1 Col: 8 Unexpected start tag (iframe). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <body>
+|     <iframe>
+|       "...<!--X->...<!--/X->..."
+
+#data
+<xmp><!--<xmp></xmp>--></xmp>
+#errors
+Line: 1 Col: 5 Unexpected start tag (xmp). Expected DOCTYPE.
+Line: 1 Col: 29 Unexpected end tag (xmp).
+#document
+| <html>
+|   <head>
+|   <body>
+|     <xmp>
+|       "<!--<xmp>"
+|     "-->"
+
+#data
+<noembed><!--<noembed></noembed>--></noembed>
+#errors
+Line: 1 Col: 9 Unexpected start tag (noembed). Expected DOCTYPE.
+Line: 1 Col: 45 Unexpected end tag (noembed).
+#document
+| <html>
+|   <head>
+|   <body>
+|     <noembed>
+|       "<!--<noembed>"
+|     "-->"
+
+#data
+<!doctype html><table>
+
+#errors
+Line 2 Col 0 Unexpected end of file. Expected table content.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       "
+"
+
+#data
+<!doctype html><table><td><span><font></span><span>
+#errors
+Line 1 Col 26 Unexpected table cell start tag (td) in the table body phase.
+Line 1 Col 45 Unexpected end tag (span).
+Line 1 Col 51 Expected closing tag. Unexpected end of file.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <tbody>
+|         <tr>
+|           <td>
+|             <span>
+|               <font>
+|             <font>
+|               <span>
+
+#data
+<!doctype html><form><table></form><form></table></form>
+#errors
+35: Stray end tag “form”.
+41: Start tag “form” seen in “table”.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <form>
+|       <table>
+|         <form>
diff --git a/libgo/go/html/testdata/webkit/tests2.dat b/libgo/go/html/testdata/webkit/tests2.dat
new file mode 100644
index 000000000..d33996e0c
--- /dev/null
+++ b/libgo/go/html/testdata/webkit/tests2.dat
@@ -0,0 +1,738 @@
+#data
+<!DOCTYPE html>Test
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     "Test"
+
+#data
+<textarea>test</div>test
+#errors
+Line: 1 Col: 10 Unexpected start tag (textarea). Expected DOCTYPE.
+Line: 1 Col: 24 Expected closing tag. Unexpected end of file.
+#document
+| <html>
+|   <head>
+|   <body>
+|     <textarea>
+|       "test</div>test"
+
+#data
+<table><td>
+#errors
+Line: 1 Col: 7 Unexpected start tag (table). Expected DOCTYPE.
+Line: 1 Col: 11 Unexpected table cell start tag (td) in the table body phase.
+Line: 1 Col: 11 Expected closing tag. Unexpected end of file.
+#document
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <tbody>
+|         <tr>
+|           <td>
+
+#data
+<table><td>test</tbody></table>
+#errors
+Line: 1 Col: 7 Unexpected start tag (table). Expected DOCTYPE.
+Line: 1 Col: 11 Unexpected table cell start tag (td) in the table body phase.
+#document
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <tbody>
+|         <tr>
+|           <td>
+|             "test"
+
+#data
+<frame>test
+#errors
+Line: 1 Col: 7 Unexpected start tag (frame). Expected DOCTYPE.
+Line: 1 Col: 7 Unexpected start tag frame. Ignored.
+#document
+| <html>
+|   <head>
+|   <body>
+|     "test"
+
+#data
+<!DOCTYPE html><frameset>test
+#errors
+Line: 1 Col: 29 Unepxected characters in the frameset phase. Characters ignored.
+Line: 1 Col: 29 Expected closing tag. Unexpected end of file.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+
+#data
+<!DOCTYPE html><frameset><!DOCTYPE html>
+#errors
+Line: 1 Col: 40 Unexpected DOCTYPE. Ignored.
+Line: 1 Col: 40 Expected closing tag. Unexpected end of file.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+
+#data
+<!DOCTYPE html><font><p><b>test</font>
+#errors
+Line: 1 Col: 38 End tag (font) violates step 1, paragraph 3 of the adoption agency algorithm.
+Line: 1 Col: 38 End tag (font) violates step 1, paragraph 3 of the adoption agency algorithm.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <font>
+|     <p>
+|       <font>
+|         <b>
+|           "test"
+
+#data
+<!DOCTYPE html><dt><div><dd>
+#errors
+Line: 1 Col: 28 Missing end tag (div, dt).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <dt>
+|       <div>
+|     <dd>
+
+#data
+<script></x
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 11 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</x"
+|   <body>
+
+#data
+<table><plaintext><td>
+#errors
+Line: 1 Col: 7 Unexpected start tag (table). Expected DOCTYPE.
+Line: 1 Col: 18 Unexpected start tag (plaintext) in table context caused voodoo mode.
+Line: 1 Col: 22 Unexpected end of file. Expected table content.
+#document
+| <html>
+|   <head>
+|   <body>
+|     <plaintext>
+|       "<td>"
+|     <table>
+
+#data
+<plaintext></plaintext>
+#errors
+Line: 1 Col: 11 Unexpected start tag (plaintext). Expected DOCTYPE.
+Line: 1 Col: 23 Expected closing tag. Unexpected end of file.
+#document
+| <html>
+|   <head>
+|   <body>
+|     <plaintext>
+|       "</plaintext>"
+
+#data
+<!DOCTYPE html><table><tr>TEST
+#errors
+Line: 1 Col: 30 Unexpected non-space characters in table context caused voodoo mode.
+Line: 1 Col: 30 Unexpected end of file. Expected table content.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     "TEST"
+|     <table>
+|       <tbody>
+|         <tr>
+
+#data
+<!DOCTYPE html><body t1=1><body t2=2><body t3=3 t4=4>
+#errors
+Line: 1 Col: 37 Unexpected start tag (body).
+Line: 1 Col: 53 Unexpected start tag (body).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     t1="1"
+|     t2="2"
+|     t3="3"
+|     t4="4"
+
+#data
+</b test
+#errors
+Line: 1 Col: 8 Unexpected end of file in attribute name.
+Line: 1 Col: 8 End tag contains unexpected attributes.
+Line: 1 Col: 8 Unexpected end tag (b). Expected DOCTYPE.
+Line: 1 Col: 8 Unexpected end tag (b) after the (implied) root element.
+#document
+| <html>
+|   <head>
+|   <body>
+
+#data
+<!DOCTYPE html></b test<b &=&>X
+#errors
+Line: 1 Col: 32 Named entity didn't end with ';'.
+Line: 1 Col: 33 End tag contains unexpected attributes.
+Line: 1 Col: 33 Unexpected end tag (b) after the (implied) root element.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     "X"
+
+#data
+<!doctypehtml><scrIPt type=text/x-foobar;baz>X</SCRipt
+#errors
+Line: 1 Col: 9 No space after literal string 'DOCTYPE'.
+Line: 1 Col: 54 Unexpected end of file in the tag name.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       type="text/x-foobar;baz"
+|       "X</SCRipt"
+|   <body>
+
+#data
+&
+#errors
+Line: 1 Col: 1 Unexpected non-space characters. Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <body>
+|     "&"
+
+#data
+&#
+#errors
+Line: 1 Col: 1 Numeric entity expected. Got end of file instead.
+Line: 1 Col: 1 Unexpected non-space characters. Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <body>
+|     "&#"
+
+#data
+&#X
+#errors
+Line: 1 Col: 3 Numeric entity expected but none found.
+Line: 1 Col: 3 Unexpected non-space characters. Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <body>
+|     "&#X"
+
+#data
+&#x
+#errors
+Line: 1 Col: 3 Numeric entity expected but none found.
+Line: 1 Col: 3 Unexpected non-space characters. Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <body>
+|     "&#x"
+
+#data
+-
+#errors
+Line: 1 Col: 4 Numeric entity didn't end with ';'.
+Line: 1 Col: 4 Unexpected non-space characters. Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <body>
+|     "-"
+
+#data
+&x-test
+#errors
+Line: 1 Col: 1 Named entity expected. Got none.
+Line: 1 Col: 1 Unexpected non-space characters. Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <body>
+|     "&x-test"
+
+#data
+<!doctypehtml><p><li>
+#errors
+Line: 1 Col: 9 No space after literal string 'DOCTYPE'.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|     <li>
+
+#data
+<!doctypehtml><p><dt>
+#errors
+Line: 1 Col: 9 No space after literal string 'DOCTYPE'.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|     <dt>
+
+#data
+<!doctypehtml><p><dd>
+#errors
+Line: 1 Col: 9 No space after literal string 'DOCTYPE'.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|     <dd>
+
+#data
+<!doctypehtml><p><form>
+#errors
+Line: 1 Col: 9 No space after literal string 'DOCTYPE'.
+Line: 1 Col: 23 Expected closing tag. Unexpected end of file.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|     <form>
+
+#data
+<!DOCTYPE html><p></P>X
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|     "X"
+
+#data
+&
+#errors
+Line: 1 Col: 4 Named entity didn't end with ';'.
+Line: 1 Col: 4 Unexpected non-space characters. Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <body>
+|     "&"
+
+#data
+&AMp;
+#errors
+Line: 1 Col: 1 Named entity expected. Got none.
+Line: 1 Col: 1 Unexpected non-space characters. Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <body>
+|     "&AMp;"
+
+#data
+<!DOCTYPE html><html><head></head><body><thisISasillyTESTelementNameToMakeSureCrazyTagNamesArePARSEDcorrectLY>
+#errors
+Line: 1 Col: 110 Expected closing tag. Unexpected end of file.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <thisisasillytestelementnametomakesurecrazytagnamesareparsedcorrectly>
+
+#data
+<!DOCTYPE html>X</body>X
+#errors
+Line: 1 Col: 24 Unexpected non-space characters in the after body phase.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     "XX"
+
+#data
+<!DOCTYPE html><!-- X
+#errors
+Line: 1 Col: 21 Unexpected end of file in comment.
+#document
+| <!DOCTYPE html>
+| <!--  X -->
+| <html>
+|   <head>
+|   <body>
+
+#data
+<!DOCTYPE html><table><caption>test TEST</caption><td>test
+#errors
+Line: 1 Col: 54 Unexpected table cell start tag (td) in the table body phase.
+Line: 1 Col: 58 Expected closing tag. Unexpected end of file.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <caption>
+|         "test TEST"
+|       <tbody>
+|         <tr>
+|           <td>
+|             "test"
+
+#data
+<!DOCTYPE html><select><option><optgroup>
+#errors
+Line: 1 Col: 41 Expected closing tag. Unexpected end of file.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+|       <option>
+|       <optgroup>
+
+#data
+<!DOCTYPE html><select><optgroup><option></optgroup><option><select><option>
+#errors
+Line: 1 Col: 68 Unexpected select start tag in the select phase treated as select end tag.
+Line: 1 Col: 76 Expected closing tag. Unexpected end of file.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+|       <optgroup>
+|         <option>
+|       <option>
+|     <option>
+
+#data
+<!DOCTYPE html><select><optgroup><option><optgroup>
+#errors
+Line: 1 Col: 51 Expected closing tag. Unexpected end of file.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+|       <optgroup>
+|         <option>
+|       <optgroup>
+
+#data
+<!DOCTYPE html><font><input><input></font>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <font>
+|       <input>
+|       <input>
+
+#data
+<!DOCTYPE html><!-- XXX - XXX -->
+#errors
+#document
+| <!DOCTYPE html>
+| <!--  XXX - XXX  -->
+| <html>
+|   <head>
+|   <body>
+
+#data
+<!DOCTYPE html><!-- XXX - XXX
+#errors
+Line: 1 Col: 29 Unexpected end of file in comment (-)
+#document
+| <!DOCTYPE html>
+| <!--  XXX - XXX -->
+| <html>
+|   <head>
+|   <body>
+
+#data
+<!DOCTYPE html><!-- XXX - XXX - XXX -->
+#errors
+#document
+| <!DOCTYPE html>
+| <!--  XXX - XXX - XXX  -->
+| <html>
+|   <head>
+|   <body>
+
+#data
+<isindex test=x name=x>
+#errors
+Line: 1 Col: 23 Unexpected start tag (isindex). Expected DOCTYPE.
+Line: 1 Col: 23 Unexpected start tag isindex. Don't use it!
+#document
+| <html>
+|   <head>
+|   <body>
+|     <form>
+|       <hr>
+|       <label>
+|         "This is a searchable index. Insert your search keywords here: "
+|         <input>
+|           name="isindex"
+|           test="x"
+|       <hr>
+
+#data
+test
+test
+#errors
+Line: 2 Col: 4 Unexpected non-space characters. Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <body>
+|     "test
+test"
+
+#data
+<!DOCTYPE html><body><title>test</body>
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       "test</body>"
+
+#data
+<!DOCTYPE html><body><title>X
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       "X"
+|     <meta>
+|       name="z"
+|     <link>
+|       rel="foo"
+|     <style>
+|       "
+x { content:"</style" } "
+
+#data
+<!DOCTYPE html><select><optgroup></optgroup></select>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+|       <optgroup>
+
+#data
+ 
+ 
+#errors
+Line: 2 Col: 1 Unexpected End of file. Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <body>
+
+#data
+<!DOCTYPE html>  <html>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+
+#data
+<!DOCTYPE html><script>
+</script>  <title>x  
+#errors
+#document
+| 
+| 
+|   
+|     
+#errors
+Line: 1 Col: 6 Unexpected start tag (head). Expected DOCTYPE.
+Line: 1 Col: 21 Unexpected start tag (script) that can be in head. Moved.
+#document
+| 
+|   
+|     
+#errors
+Line: 1 Col: 6 Unexpected start tag (head). Expected DOCTYPE.
+Line: 1 Col: 28 Unexpected start tag (style) that can be in head. Moved.
+#document
+| 
+|   
+|     
+#errors
+Line: 1 Col: 6 Unexpected start tag (head). Expected DOCTYPE.
+#document
+| 
+|   
+|   
+|   
+|     "x"
+|     x
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+Line: 1 Col: 22 Unexpected end of file. Expected end tag (style).
+#document
+| 
+|   
+|      --> x
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+#document
+| 
+|   
+|     x
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+#document
+| 
+|   
+|     x
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+#document
+| 
+|   
+|     x
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+#document
+| 
+|   
+|      

+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|