MDL-65763 lib: Upgrade MixMind DB reader lib to 1.4.1

This commit is contained in:
Mihail Geshoski
2019-07-12 08:17:46 +08:00
parent e952e8a425
commit b55260fcfb
4 changed files with 217 additions and 185 deletions
+47 -37
View File
@@ -29,15 +29,16 @@ class Reader
* be a valid MaxMind DB file such as a GeoIp2 database file.
*
* @param string $database
* the MaxMind DB file to use.
* @throws \InvalidArgumentException for invalid database path or unknown arguments
* the MaxMind DB file to use
*
* @throws \InvalidArgumentException for invalid database path or unknown arguments
* @throws \MaxMind\Db\Reader\InvalidDatabaseException
* if the database is invalid or there is an error reading
* from it.
* if the database is invalid or there is an error reading
* from it
*/
public function __construct($database)
{
if (func_num_args() != 1) {
if (\func_num_args() !== 1) {
throw new \InvalidArgumentException(
'The constructor takes exactly one argument.'
);
@@ -75,23 +76,25 @@ class Reader
* Looks up the <code>address</code> in the MaxMind DB.
*
* @param string $ipAddress
* the IP address to look up.
* @return array the record for the IP address.
* @throws \BadMethodCallException if this method is called on a closed database.
* @throws \InvalidArgumentException if something other than a single IP address is passed to the method.
* the IP address to look up
*
* @throws \BadMethodCallException if this method is called on a closed database
* @throws \InvalidArgumentException if something other than a single IP address is passed to the method
* @throws InvalidDatabaseException
* if the database is invalid or there is an error reading
* from it.
* if the database is invalid or there is an error reading
* from it
*
* @return array the record for the IP address
*/
public function get($ipAddress)
{
if (func_num_args() != 1) {
if (\func_num_args() !== 1) {
throw new \InvalidArgumentException(
'Method takes exactly one argument.'
);
}
if (!is_resource($this->fileHandle)) {
if (!\is_resource($this->fileHandle)) {
throw new \BadMethodCallException(
'Attempt to read from a closed MaxMind DB.'
);
@@ -103,16 +106,17 @@ class Reader
);
}
if ($this->metadata->ipVersion == 4 && strrpos($ipAddress, ':')) {
if ($this->metadata->ipVersion === 4 && strrpos($ipAddress, ':')) {
throw new \InvalidArgumentException(
"Error looking up $ipAddress. You attempted to look up an"
. " IPv6 address in an IPv4-only database."
. ' IPv6 address in an IPv4-only database.'
);
}
$pointer = $this->findAddressInTree($ipAddress);
if ($pointer == 0) {
if ($pointer === 0) {
return null;
}
return $this->resolveDataPointer($pointer);
}
@@ -121,13 +125,13 @@ class Reader
// XXX - could simplify. Done as a byte array to ease porting
$rawAddress = array_merge(unpack('C*', inet_pton($ipAddress)));
$bitCount = count($rawAddress) * 8;
$bitCount = \count($rawAddress) * 8;
// The first node of the tree is always node 0, at the beginning of the
// value
$node = $this->startNode($bitCount);
for ($i = 0; $i < $bitCount; $i++) {
for ($i = 0; $i < $bitCount; ++$i) {
if ($node >= $this->metadata->nodeCount) {
break;
}
@@ -136,22 +140,21 @@ class Reader
$node = $this->readNode($node, $bit);
}
if ($node == $this->metadata->nodeCount) {
if ($node === $this->metadata->nodeCount) {
// Record is empty
return 0;
} elseif ($node > $this->metadata->nodeCount) {
// Record is a data pointer
return $node;
}
throw new InvalidDatabaseException("Something bad happened");
throw new InvalidDatabaseException('Something bad happened');
}
private function startNode($length)
{
// Check if we are looking up an IPv4 address in an IPv6 tree. If this
// is the case, we can skip over the first 96 nodes.
if ($this->metadata->ipVersion == 6 && $length == 32) {
if ($this->metadata->ipVersion === 6 && $length === 32) {
return $this->ipV4StartNode();
}
// The first node of the tree is always node 0, at the beginning of the
@@ -163,19 +166,20 @@ class Reader
{
// This is a defensive check. There is no reason to call this when you
// have an IPv4 tree.
if ($this->metadata->ipVersion == 4) {
if ($this->metadata->ipVersion === 4) {
return 0;
}
if ($this->ipV4Start != 0) {
if ($this->ipV4Start) {
return $this->ipV4Start;
}
$node = 0;
for ($i = 0; $i < 96 && $node < $this->metadata->nodeCount; $i++) {
for ($i = 0; $i < 96 && $node < $this->metadata->nodeCount; ++$i) {
$node = $this->readNode($node, 0);
}
$this->ipV4Start = $node;
return $node;
}
@@ -188,21 +192,24 @@ class Reader
case 24:
$bytes = Util::read($this->fileHandle, $baseOffset + $index * 3, 3);
list(, $node) = unpack('N', "\x00" . $bytes);
return $node;
case 28:
$middleByte = Util::read($this->fileHandle, $baseOffset + 3, 1);
list(, $middle) = unpack('C', $middleByte);
if ($index == 0) {
if ($index === 0) {
$middle = (0xF0 & $middle) >> 4;
} else {
$middle = 0x0F & $middle;
}
$bytes = Util::read($this->fileHandle, $baseOffset + $index * 4, 3);
list(, $node) = unpack('N', chr($middle) . $bytes);
list(, $node) = unpack('N', \chr($middle) . $bytes);
return $node;
case 32:
$bytes = Util::read($this->fileHandle, $baseOffset + $index * 4, 4);
list(, $node) = unpack('N', $bytes);
return $node;
default:
throw new InvalidDatabaseException(
@@ -223,6 +230,7 @@ class Reader
}
list($data) = $this->decoder->decode($resolved);
return $data;
}
@@ -241,14 +249,15 @@ class Reader
$metadataMaxLengthExcludingMarker
= min(self::$METADATA_MAX_SIZE, $fileSize) - $markerLength;
for ($i = 0; $i <= $metadataMaxLengthExcludingMarker; $i++) {
for ($j = 0; $j < $markerLength; $j++) {
for ($i = 0; $i <= $metadataMaxLengthExcludingMarker; ++$i) {
for ($j = 0; $j < $markerLength; ++$j) {
fseek($handle, $fileSize - $i - $j - 1);
$matchBit = fgetc($handle);
if ($matchBit != $marker[$markerLength - $j - 1]) {
if ($matchBit !== $marker[$markerLength - $j - 1]) {
continue 2;
}
}
return $fileSize - $i;
}
throw new InvalidDatabaseException(
@@ -258,13 +267,14 @@ class Reader
}
/**
* @throws \InvalidArgumentException if arguments are passed to the method.
* @throws \BadMethodCallException if the database has been closed.
* @return Metadata object for the database.
* @throws \InvalidArgumentException if arguments are passed to the method
* @throws \BadMethodCallException if the database has been closed
*
* @return Metadata object for the database
*/
public function metadata()
{
if (func_num_args()) {
if (\func_num_args()) {
throw new \InvalidArgumentException(
'Method takes no arguments.'
);
@@ -272,7 +282,7 @@ class Reader
// Not technically required, but this makes it consistent with
// C extension and it allows us to change our implementation later.
if (!is_resource($this->fileHandle)) {
if (!\is_resource($this->fileHandle)) {
throw new \BadMethodCallException(
'Attempt to read from a closed MaxMind DB.'
);
@@ -285,11 +295,11 @@ class Reader
* Closes the MaxMind DB and returns resources to the system.
*
* @throws \Exception
* if an I/O error occurs.
* if an I/O error occurs
*/
public function close()
{
if (!is_resource($this->fileHandle)) {
if (!\is_resource($this->fileHandle)) {
throw new \BadMethodCallException(
'Attempt to close a closed MaxMind DB.'
);
+161 -129
View File
@@ -2,36 +2,35 @@
namespace MaxMind\Db\Reader;
use MaxMind\Db\Reader\InvalidDatabaseException;
use MaxMind\Db\Reader\Util;
// @codingStandardsIgnoreLine
// We subtract 1 from the log to protect against precision loss.
\define(__NAMESPACE__ . '\_MM_MAX_INT_BYTES', (log(PHP_INT_MAX, 2) - 1) / 8);
class Decoder
{
private $fileStream;
private $pointerBase;
private $pointerBaseByteSize;
// This is only used for unit testing
private $pointerTestHack;
private $switchByteOrder;
private $types = array(
0 => 'extended',
1 => 'pointer',
2 => 'utf8_string',
3 => 'double',
4 => 'bytes',
5 => 'uint16',
6 => 'uint32',
7 => 'map',
8 => 'int32',
9 => 'uint64',
10 => 'uint128',
11 => 'array',
12 => 'container',
13 => 'end_marker',
14 => 'boolean',
15 => 'float',
);
const _EXTENDED = 0;
const _POINTER = 1;
const _UTF8_STRING = 2;
const _DOUBLE = 3;
const _BYTES = 4;
const _UINT16 = 5;
const _UINT32 = 6;
const _MAP = 7;
const _INT32 = 8;
const _UINT64 = 9;
const _UINT128 = 10;
const _ARRAY = 11;
const _CONTAINER = 12;
const _END_MARKER = 13;
const _BOOLEAN = 14;
const _FLOAT = 15;
public function __construct(
$fileStream,
@@ -40,57 +39,57 @@ class Decoder
) {
$this->fileStream = $fileStream;
$this->pointerBase = $pointerBase;
$this->pointerBaseByteSize = $pointerBase > 0 ? log($pointerBase, 2) / 8 : 0;
$this->pointerTestHack = $pointerTestHack;
$this->switchByteOrder = $this->isPlatformLittleEndian();
}
public function decode($offset)
{
list(, $ctrlByte) = unpack(
'C',
Util::read($this->fileStream, $offset, 1)
);
$offset++;
++$offset;
$type = $this->types[$ctrlByte >> 5];
$type = $ctrlByte >> 5;
// Pointers are a special case, we don't read the next $size bytes, we
// use the size to determine the length of the pointer and then follow
// it.
if ($type == 'pointer') {
if ($type === self::_POINTER) {
list($pointer, $offset) = $this->decodePointer($ctrlByte, $offset);
// for unit testing
if ($this->pointerTestHack) {
return array($pointer);
return [$pointer];
}
list($result) = $this->decode($pointer);
return array($result, $offset);
return [$result, $offset];
}
if ($type == 'extended') {
if ($type === self::_EXTENDED) {
list(, $nextByte) = unpack(
'C',
Util::read($this->fileStream, $offset, 1)
);
$typeNum = $nextByte + 7;
$type = $nextByte + 7;
if ($typeNum < 8) {
if ($type < 8) {
throw new InvalidDatabaseException(
"Something went horribly wrong in the decoder. An extended type "
. "resolved to a type number < 8 ("
. $this->types[$typeNum]
. ")"
'Something went horribly wrong in the decoder. An extended type '
. 'resolved to a type number < 8 ('
. $type
. ')'
);
}
$type = $this->types[$typeNum];
$offset++;
++$offset;
}
list($size, $offset) = $this->sizeFromCtrlByte($ctrlByte, $offset);
@@ -101,45 +100,45 @@ class Decoder
private function decodeByType($type, $offset, $size)
{
switch ($type) {
case 'map':
case self::_MAP:
return $this->decodeMap($size, $offset);
case 'array':
case self::_ARRAY:
return $this->decodeArray($size, $offset);
case 'boolean':
return array($this->decodeBoolean($size), $offset);
case self::_BOOLEAN:
return [$this->decodeBoolean($size), $offset];
}
$newOffset = $offset + $size;
$bytes = Util::read($this->fileStream, $offset, $size);
switch ($type) {
case 'utf8_string':
return array($this->decodeString($bytes), $newOffset);
case 'double':
case self::_BYTES:
case self::_UTF8_STRING:
return [$bytes, $newOffset];
case self::_DOUBLE:
$this->verifySize(8, $size);
return array($this->decodeDouble($bytes), $newOffset);
case 'float':
return [$this->decodeDouble($bytes), $newOffset];
case self::_FLOAT:
$this->verifySize(4, $size);
return array($this->decodeFloat($bytes), $newOffset);
case 'bytes':
return array($bytes, $newOffset);
case 'uint16':
case 'uint32':
return array($this->decodeUint($bytes), $newOffset);
case 'int32':
return array($this->decodeInt32($bytes), $newOffset);
case 'uint64':
case 'uint128':
return array($this->decodeBigUint($bytes, $size), $newOffset);
return [$this->decodeFloat($bytes), $newOffset];
case self::_INT32:
return [$this->decodeInt32($bytes, $size), $newOffset];
case self::_UINT16:
case self::_UINT32:
case self::_UINT64:
case self::_UINT128:
return [$this->decodeUint($bytes, $size), $newOffset];
default:
throw new InvalidDatabaseException(
"Unknown or unexpected type: " . $type
'Unknown or unexpected type: ' . $type
);
}
}
private function verifySize($expected, $actual)
{
if ($expected != $actual) {
if ($expected !== $actual) {
throw new InvalidDatabaseException(
"The MaxMind DB file's data section contains bad data (unknown data type or corrupt data)"
);
@@ -148,63 +147,83 @@ class Decoder
private function decodeArray($size, $offset)
{
$array = array();
$array = [];
for ($i = 0; $i < $size; $i++) {
for ($i = 0; $i < $size; ++$i) {
list($value, $offset) = $this->decode($offset);
array_push($array, $value);
}
return array($array, $offset);
return [$array, $offset];
}
private function decodeBoolean($size)
{
return $size == 0 ? false : true;
return $size === 0 ? false : true;
}
private function decodeDouble($bits)
{
// XXX - Assumes IEEE 754 double on platform
// This assumes IEEE 754 doubles, but most (all?) modern platforms
// use them.
//
// We are not using the "E" format as that was only added in
// 7.0.15 and 7.1.1. As such, we must switch byte order on
// little endian machines.
list(, $double) = unpack('d', $this->maybeSwitchByteOrder($bits));
return $double;
}
private function decodeFloat($bits)
{
// XXX - Assumes IEEE 754 floats on platform
// This assumes IEEE 754 floats, but most (all?) modern platforms
// use them.
//
// We are not using the "G" format as that was only added in
// 7.0.15 and 7.1.1. As such, we must switch byte order on
// little endian machines.
list(, $float) = unpack('f', $this->maybeSwitchByteOrder($bits));
return $float;
}
private function decodeInt32($bytes)
private function decodeInt32($bytes, $size)
{
$bytes = $this->zeroPadLeft($bytes, 4);
switch ($size) {
case 0:
return 0;
case 1:
case 2:
case 3:
$bytes = str_pad($bytes, 4, "\x00", STR_PAD_LEFT);
break;
case 4:
break;
default:
throw new InvalidDatabaseException(
"The MaxMind DB file's data section contains bad data (unknown data type or corrupt data)"
);
}
list(, $int) = unpack('l', $this->maybeSwitchByteOrder($bytes));
return $int;
}
private function decodeMap($size, $offset)
{
$map = [];
$map = array();
for ($i = 0; $i < $size; $i++) {
for ($i = 0; $i < $size; ++$i) {
list($key, $offset) = $this->decode($offset);
list($value, $offset) = $this->decode($offset);
$map[$key] = $value;
}
return array($map, $offset);
return [$map, $offset];
}
private $pointerValueOffset = array(
1 => 0,
2 => 2048,
3 => 526336,
4 => 0,
);
private function decodePointer($ctrlByte, $offset)
{
$pointerSize = (($ctrlByte >> 3) & 0x3) + 1;
@@ -212,87 +231,99 @@ class Decoder
$buffer = Util::read($this->fileStream, $offset, $pointerSize);
$offset = $offset + $pointerSize;
$packed = $pointerSize == 4
? $buffer
: (pack('C', $ctrlByte & 0x7)) . $buffer;
switch ($pointerSize) {
case 1:
$packed = (pack('C', $ctrlByte & 0x7)) . $buffer;
list(, $pointer) = unpack('n', $packed);
$pointer += $this->pointerBase;
break;
case 2:
$packed = "\x00" . (pack('C', $ctrlByte & 0x7)) . $buffer;
list(, $pointer) = unpack('N', $packed);
$pointer += $this->pointerBase + 2048;
break;
case 3:
$packed = (pack('C', $ctrlByte & 0x7)) . $buffer;
$unpacked = $this->decodeUint($packed);
$pointer = $unpacked + $this->pointerBase
+ $this->pointerValueOffset[$pointerSize];
// It is safe to use 'N' here, even on 32 bit machines as the
// first bit is 0.
list(, $pointer) = unpack('N', $packed);
$pointer += $this->pointerBase + 526336;
break;
case 4:
// We cannot use unpack here as we might overflow on 32 bit
// machines
$pointerOffset = $this->decodeUint($buffer, $pointerSize);
return array($pointer, $offset);
$byteLength = $pointerSize + $this->pointerBaseByteSize;
if ($byteLength <= _MM_MAX_INT_BYTES) {
$pointer = $pointerOffset + $this->pointerBase;
} elseif (\extension_loaded('gmp')) {
$pointer = gmp_strval(gmp_add($pointerOffset, $this->pointerBase));
} elseif (\extension_loaded('bcmath')) {
$pointer = bcadd($pointerOffset, $this->pointerBase);
} else {
throw new \RuntimeException(
'The gmp or bcmath extension must be installed to read this database.'
);
}
}
return [$pointer, $offset];
}
private function decodeUint($bytes)
private function decodeUint($bytes, $byteLength)
{
list(, $int) = unpack('N', $this->zeroPadLeft($bytes, 4));
return $int;
}
private function decodeBigUint($bytes, $byteLength)
{
$maxUintBytes = log(PHP_INT_MAX, 2) / 8;
if ($byteLength == 0) {
if ($byteLength === 0) {
return 0;
}
$numberOfLongs = ceil($byteLength / 4);
$paddedLength = $numberOfLongs * 4;
$paddedBytes = $this->zeroPadLeft($bytes, $paddedLength);
$unpacked = array_merge(unpack("N$numberOfLongs", $paddedBytes));
$integer = 0;
// 2^32
$twoTo32 = '4294967296';
for ($i = 0; $i < $byteLength; ++$i) {
$part = \ord($bytes[$i]);
foreach ($unpacked as $part) {
// We only use gmp or bcmath if the final value is too big
if ($byteLength <= $maxUintBytes) {
$integer = ($integer << 32) + $part;
} elseif (extension_loaded('gmp')) {
$integer = gmp_strval(gmp_add(gmp_mul($integer, $twoTo32), $part));
} elseif (extension_loaded('bcmath')) {
$integer = bcadd(bcmul($integer, $twoTo32), $part);
if ($byteLength <= _MM_MAX_INT_BYTES) {
$integer = ($integer << 8) + $part;
} elseif (\extension_loaded('gmp')) {
$integer = gmp_strval(gmp_add(gmp_mul($integer, 256), $part));
} elseif (\extension_loaded('bcmath')) {
$integer = bcadd(bcmul($integer, 256), $part);
} else {
throw new \RuntimeException(
'The gmp or bcmath extension must be installed to read this database.'
);
}
}
return $integer;
}
private function decodeString($bytes)
{
// XXX - NOOP. As far as I know, the end user has to explicitly set the
// encoding in PHP. Strings are just bytes.
return $bytes;
return $integer;
}
private function sizeFromCtrlByte($ctrlByte, $offset)
{
$size = $ctrlByte & 0x1f;
$bytesToRead = $size < 29 ? 0 : $size - 28;
$bytes = Util::read($this->fileStream, $offset, $bytesToRead);
$decoded = $this->decodeUint($bytes);
if ($size == 29) {
$size = 29 + $decoded;
} elseif ($size == 30) {
$size = 285 + $decoded;
if ($size < 29) {
return [$size, $offset];
}
$bytesToRead = $size - 28;
$bytes = Util::read($this->fileStream, $offset, $bytesToRead);
if ($size === 29) {
$size = 29 + \ord($bytes);
} elseif ($size === 30) {
list(, $adjust) = unpack('n', $bytes);
$size = 285 + $adjust;
} elseif ($size > 30) {
$size = ($decoded & (0x0FFFFFFF >> (32 - (8 * $bytesToRead))))
list(, $adjust) = unpack('N', "\x00" . $bytes);
$size = ($adjust & (0x0FFFFFFF >> (32 - (8 * $bytesToRead))))
+ 65821;
}
return array($size, $offset + $bytesToRead);
}
private function zeroPadLeft($content, $desiredLength)
{
return str_pad($content, $desiredLength, "\x00", STR_PAD_LEFT);
return [$size, $offset + $bytesToRead];
}
private function maybeSwitchByteOrder($bytes)
@@ -304,6 +335,7 @@ class Decoder
{
$testint = 0x00FF;
$packed = pack('S', $testint);
return $testint === current(unpack('v', $packed));
}
}
+6 -14
View File
@@ -5,35 +5,27 @@ namespace MaxMind\Db\Reader;
/**
* This class provides the metadata for the MaxMind DB file.
*
* @property integer nodeCount This is an unsigned 32-bit integer indicating
* @property int nodeCount This is an unsigned 32-bit integer indicating
* the number of nodes in the search tree.
*
* @property integer recordSize This is an unsigned 16-bit integer. It
* @property int recordSize This is an unsigned 16-bit integer. It
* indicates the number of bits in a record in the search tree. Note that each
* node consists of two records.
*
* @property integer ipVersion This is an unsigned 16-bit integer which is
* @property int ipVersion This is an unsigned 16-bit integer which is
* always 4 or 6. It indicates whether the database contains IPv4 or IPv6
* address data.
*
* @property string databaseType This is a string that indicates the structure
* of each data record associated with an IP address. The actual definition of
* these structures is left up to the database creator.
*
* @property array languages An array of strings, each of which is a language
* code. A given record may contain data items that have been localized to
* some or all of these languages. This may be undefined.
*
* @property integer binaryFormatMajorVersion This is an unsigned 16-bit
* @property int binaryFormatMajorVersion This is an unsigned 16-bit
* integer indicating the major version number for the database's binary
* format.
*
* @property integer binaryFormatMinorVersion This is an unsigned 16-bit
* @property int binaryFormatMinorVersion This is an unsigned 16-bit
* integer indicating the minor version number for the database's binary format.
*
* @property integer buildEpoch This is an unsigned 64-bit integer that
* @property int buildEpoch This is an unsigned 64-bit integer that
* contains the database build timestamp as a Unix epoch value.
*
* @property array description This key will always point to a map
* (associative array). The keys of that map will be language codes, and the
* values will be a description in that language as a UTF-8 string. May be
+3 -5
View File
@@ -2,16 +2,14 @@
namespace MaxMind\Db\Reader;
use MaxMind\Db\Reader\InvalidDatabaseException;
class Util
{
public static function read($stream, $offset, $numberOfBytes)
{
if ($numberOfBytes == 0) {
if ($numberOfBytes === 0) {
return '';
}
if (fseek($stream, $offset) == 0) {
if (fseek($stream, $offset) === 0) {
$value = fread($stream, $numberOfBytes);
// We check that the number of bytes read is equal to the number
@@ -22,7 +20,7 @@ class Util
}
}
throw new InvalidDatabaseException(
"The MaxMind DB file contains bad data"
'The MaxMind DB file contains bad data'
);
}
}