  void Compute(OpKernelContext* const context) override {
    // Read float features list;
    OpInputList float_features_list;
    OP_REQUIRES_OK(
        context, context->input_list(kFloatFeaturesName, &float_features_list));
    OpInputList bucket_boundaries_list;
    OP_REQUIRES_OK(context, context->input_list(kBucketBoundariesName,
                                                &bucket_boundaries_list));
    OP_REQUIRES(context,
                tensorflow::TensorShapeUtils::IsVector(
                    bucket_boundaries_list[0].shape()),
                errors::InvalidArgument(
                    strings::Printf("Buckets should be flat vectors.")));
    OpOutputList buckets_list;
    OP_REQUIRES_OK(context, context->output_list(kBucketsName, &buckets_list));

    auto do_quantile_get_quantiles = [&](const int64_t begin,
                                         const int64_t end) {
      // Iterating over all resources
      for (int64_t feature_idx = begin; feature_idx < end; feature_idx++) {
        const Tensor& values_tensor = float_features_list[feature_idx];
        const int64_t num_values = values_tensor.dim_size(0);

        Tensor* output_t = nullptr;
        OP_REQUIRES_OK(context,
                       buckets_list.allocate(
                           feature_idx, TensorShape({num_values}), &output_t));
        auto output = output_t->flat<int32>();

        const std::vector<float>& bucket_boundaries_vector =
            GetBuckets(feature_idx, bucket_boundaries_list);
        auto flat_values = values_tensor.flat<float>();
        const auto& iter_begin = bucket_boundaries_vector.begin();
        const auto& iter_end = bucket_boundaries_vector.end();
        for (int64_t instance = 0; instance < num_values; instance++) {
          if (iter_begin == iter_end) {
            output(instance) = 0;
            continue;
          }
          const float value = flat_values(instance);
          auto bucket_iter = std::lower_bound(iter_begin, iter_end, value);
          if (bucket_iter == iter_end) {
            --bucket_iter;
          }
          const int32_t bucket = static_cast<int32>(bucket_iter - iter_begin);
          // Bucket id.
          output(instance) = bucket;
        }
      }
    };

    // TODO(tanzheny): comment on the magic number.
    const int64_t kCostPerUnit = 500 * num_features_;
    const DeviceBase::CpuWorkerThreads& worker_threads =
        *context->device()->tensorflow_cpu_worker_threads();
    Shard(worker_threads.num_threads, worker_threads.workers, num_features_,
          kCostPerUnit, do_quantile_get_quantiles);
  }


  explicit BoostedTreesQuantileStreamResourceGetBucketBoundariesOp(
      OpKernelConstruction* const context)
      : OpKernel(context) {
    OP_REQUIRES_OK(context, context->GetAttr(kNumFeaturesName, &num_features_));
  }

Ensure num_streams >= 0 in tf.raw_ops.BoostedTreesCreateQuantileStreamResource

PiperOrigin-RevId: 387452765
Change-Id: I9990c760e177fabca6a3b9b4612ceeaeeba51495


  void Compute(OpKernelContext* const context) override {
    // Read float features list;
    OpInputList float_features_list;
    OP_REQUIRES_OK(
        context, context->input_list(kFloatFeaturesName, &float_features_list));

    // Parse example weights and get batch size.
    const Tensor* example_weights_t;
    OP_REQUIRES_OK(context,
                   context->input(kExampleWeightsName, &example_weights_t));
    DCHECK(float_features_list.size() > 0) << "Got empty feature list";
    auto example_weights = example_weights_t->flat<float>();
    const int64_t weight_size = example_weights.size();
    const int64_t batch_size = float_features_list[0].flat<float>().size();
    OP_REQUIRES(
        context, weight_size == 1 || weight_size == batch_size,
        errors::InvalidArgument(strings::Printf(
            "Weights should be a single value or same size as features.")));
    const Tensor* epsilon_t;
    OP_REQUIRES_OK(context, context->input(kEpsilonName, &epsilon_t));
    float epsilon = epsilon_t->scalar<float>()();

    OpOutputList summaries_output_list;
    OP_REQUIRES_OK(
        context, context->output_list(kSummariesName, &summaries_output_list));

    auto do_quantile_summary_gen = [&](const int64_t begin, const int64_t end) {
      // Iterating features.
      for (int64_t index = begin; index < end; index++) {
        const auto feature_values = float_features_list[index].flat<float>();
        QuantileStream stream(epsilon, batch_size + 1);
        // Run quantile summary generation.
        for (int64_t j = 0; j < batch_size; j++) {
          stream.PushEntry(feature_values(j), (weight_size > 1)
                                                  ? example_weights(j)
                                                  : example_weights(0));
        }
        stream.Finalize();
        const auto summary_entry_list = stream.GetFinalSummary().GetEntryList();
        Tensor* output_t;
        OP_REQUIRES_OK(
            context,
            summaries_output_list.allocate(
                index,
                TensorShape({static_cast<int64>(summary_entry_list.size()), 4}),
                &output_t));
        auto output = output_t->matrix<float>();
        for (auto row = 0; row < summary_entry_list.size(); row++) {
          const auto& entry = summary_entry_list[row];
          output(row, 0) = entry.value;
          output(row, 1) = entry.weight;
          output(row, 2) = entry.min_rank;
          output(row, 3) = entry.max_rank;
        }
      }
    };
    // TODO(tanzheny): comment on the magic number.
    const int64_t kCostPerUnit = 500 * batch_size;
    const DeviceBase::CpuWorkerThreads& worker_threads =
        *context->device()->tensorflow_cpu_worker_threads();
    Shard(worker_threads.num_threads, worker_threads.workers, num_features_,
          kCostPerUnit, do_quantile_summary_gen);
  }


  explicit BoostedTreesQuantileStreamResourceFlushOp(
      OpKernelConstruction* const context)
      : OpKernel(context) {
    OP_REQUIRES_OK(context,
                   context->GetAttr(kGenerateQuantiles, &generate_quantiles_));
  }


std::vector<float> GenerateBoundaries(const QuantileStream& stream,
                                      const int64_t num_boundaries) {
  std::vector<float> boundaries = stream.GenerateBoundaries(num_boundaries);

  // Uniquify elements as we may get dupes.
  auto end_it = std::unique(boundaries.begin(), boundaries.end());
  boundaries.resize(std::distance(boundaries.begin(), end_it));
  return boundaries;
}


std::vector<float> GetBuckets(const int32_t feature,
                              const OpInputList& buckets_list) {
  const auto& buckets = buckets_list[feature].flat<float>();
  std::vector<float> buckets_vector(buckets.data(),
                                    buckets.data() + buckets.size());
  return buckets_vector;
}


  void Compute(OpKernelContext* const context) override {
    ResourceHandle handle;
    OP_REQUIRES_OK(context,
                   HandleFromInput(context, kResourceHandleName, &handle));
    core::RefCountPtr<QuantileStreamResource> stream_resource;
    OP_REQUIRES_OK(context, LookupResource(context, handle, &stream_resource));
    // Remove the reference at the end of this scope.
    mutex_lock l(*stream_resource->mutex());

    OpOutputList summaries_output_list;
    OP_REQUIRES_OK(
        context, context->output_list(kSummariesName, &summaries_output_list));

    auto do_quantile_summary_gen = [&](const int64_t begin, const int64_t end) {
      // Iterating features.
      for (int64_t index = begin; index < end; index++) {
        QuantileStream* stream = stream_resource->stream(index);
        stream->Finalize();

        const auto summary_list = stream->GetFinalSummary().GetEntryList();
        Tensor* output_t;
        const int64_t summary_list_size =
            static_cast<int64>(summary_list.size());
        OP_REQUIRES_OK(context, summaries_output_list.allocate(
                                    index, TensorShape({summary_list_size, 4}),
                                    &output_t));
        auto output = output_t->matrix<float>();
        for (auto row = 0; row < summary_list_size; row++) {
          const auto& entry = summary_list[row];
          output(row, 0) = entry.value;
          output(row, 1) = entry.weight;
          output(row, 2) = entry.min_rank;
          output(row, 3) = entry.max_rank;
        }
      }
    };
    // TODO(tanzheny): comment on the magic number.
    const int64_t kCostPerUnit = 500 * num_features_;
    const DeviceBase::CpuWorkerThreads& worker_threads =
        *context->device()->tensorflow_cpu_worker_threads();
    Shard(worker_threads.num_threads, worker_threads.workers, num_features_,
          kCostPerUnit, do_quantile_summary_gen);
    stream_resource->ResetStreams();
  }


  explicit BoostedTreesQuantileStreamResourceAddSummariesOp(
      OpKernelConstruction* const context)
      : OpKernel(context) {}


  void Compute(OpKernelContext* const context) override {
    ResourceHandle handle;
    OP_REQUIRES_OK(context,
                   HandleFromInput(context, kResourceHandleName, &handle));
    core::RefCountPtr<QuantileStreamResource> stream_resource;
    // Create a reference to the underlying resource using the handle.
    OP_REQUIRES_OK(context, LookupResource(context, handle, &stream_resource));
    // Remove the reference at the end of this scope.
    mutex_lock l(*stream_resource->mutex());

    const int64_t num_streams = stream_resource->num_streams();
    CHECK_EQ(num_features_, num_streams);
    OpOutputList bucket_boundaries_list;
    OP_REQUIRES_OK(context, context->output_list(kBucketBoundariesName,
                                                 &bucket_boundaries_list));

    auto do_quantile_get_buckets = [&](const int64_t begin, const int64_t end) {
      // Iterating over all streams.
      for (int64_t stream_idx = begin; stream_idx < end; stream_idx++) {
        const auto& boundaries = stream_resource->boundaries(stream_idx);
        Tensor* bucket_boundaries_t = nullptr;
        OP_REQUIRES_OK(context,
                       bucket_boundaries_list.allocate(
                           stream_idx, {static_cast<int64>(boundaries.size())},
                           &bucket_boundaries_t));
        auto* quantiles_flat = bucket_boundaries_t->flat<float>().data();
        memcpy(quantiles_flat, boundaries.data(),
               sizeof(float) * boundaries.size());
      }
    };

    // TODO(tanzheny): comment on the magic number.
    const int64_t kCostPerUnit = 500 * num_streams;
    const DeviceBase::CpuWorkerThreads& worker_threads =
        *context->device()->tensorflow_cpu_worker_threads();
    Shard(worker_threads.num_threads, worker_threads.workers, num_streams,
          kCostPerUnit, do_quantile_get_buckets);
  }


  void Compute(OpKernelContext* context) override {
    // Only create one, if one does not exist already. Report status for all
    // other exceptions. If one already exists, it unrefs the new one.
    // An epsilon value of zero could cause performance issues and is therefore,
    // disallowed.
    const Tensor* epsilon_t;
    OP_REQUIRES_OK(context, context->input(kEpsilonName, &epsilon_t));
    float epsilon = epsilon_t->scalar<float>()();
    OP_REQUIRES(
        context, epsilon > 0,
        errors::InvalidArgument("An epsilon value of zero is not allowed."));

    const Tensor* num_streams_t;
    OP_REQUIRES_OK(context, context->input(kNumStreamsName, &num_streams_t));
    int64_t num_streams = num_streams_t->scalar<int64>()();
    OP_REQUIRES(context, num_streams >= 0,
                errors::InvalidArgument(
                    "Num_streams input cannot be a negative integer"));

    auto result =
        new QuantileStreamResource(epsilon, max_elements_, num_streams);
    auto status = CreateResource(context, HandleFromInput(context, 0), result);
    if (!status.ok() && status.code() != tensorflow::error::ALREADY_EXISTS) {
      OP_REQUIRES(context, false, status);
    }
  }


  void Compute(OpKernelContext* context) override {
    core::RefCountPtr<QuantileStreamResource> streams_resource;
    // Create a reference to the underlying resource using the handle.
    OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0),
                                           &streams_resource));
    // Remove the reference at the end of this scope.
    mutex_lock l(*streams_resource->mutex());

    OpInputList bucket_boundaries_list;
    OP_REQUIRES_OK(context, context->input_list(kBucketBoundariesName,
                                                &bucket_boundaries_list));

    auto do_quantile_deserialize = [&](const int64_t begin, const int64_t end) {
      // Iterating over all streams.
      for (int64_t stream_idx = begin; stream_idx < end; stream_idx++) {
        const Tensor& bucket_boundaries_t = bucket_boundaries_list[stream_idx];
        const auto& bucket_boundaries = bucket_boundaries_t.vec<float>();
        std::vector<float> result;
        result.reserve(bucket_boundaries.size());
        for (size_t i = 0; i < bucket_boundaries.size(); ++i) {
          result.push_back(bucket_boundaries(i));
        }
        streams_resource->set_boundaries(result, stream_idx);
      }
    };

    // TODO(tanzheny): comment on the magic number.
    const int64_t kCostPerUnit = 500 * num_features_;
    const DeviceBase::CpuWorkerThreads& worker_threads =
        *context->device()->tensorflow_cpu_worker_threads();
    Shard(worker_threads.num_threads, worker_threads.workers, num_features_,
          kCostPerUnit, do_quantile_deserialize);
  }


  explicit BoostedTreesQuantileStreamResourceDeserializeOp(
      OpKernelConstruction* const context)
      : OpKernel(context) {
    OP_REQUIRES_OK(context, context->GetAttr(kNumStreamsName, &num_features_));
  }


std::vector<float> GenerateQuantiles(const QuantileStream& stream,
                                     const int64_t num_quantiles) {
  // Do not de-dup boundaries. Exactly num_quantiles+1 boundary values
  // will be returned.
  std::vector<float> boundaries = stream.GenerateQuantiles(num_quantiles - 1);
  CHECK_EQ(boundaries.size(), num_quantiles);
  return boundaries;
}


  explicit BoostedTreesBucketizeOp(OpKernelConstruction* const context)
      : OpKernel(context) {
    OP_REQUIRES_OK(context, context->GetAttr(kNumFeaturesName, &num_features_));
  }


  void Compute(OpKernelContext* context) override {
    ResourceHandle handle;
    OP_REQUIRES_OK(context,
                   HandleFromInput(context, kResourceHandleName, &handle));
    core::RefCountPtr<QuantileStreamResource> stream_resource;
    // Create a reference to the underlying resource using the handle.
    OP_REQUIRES_OK(context, LookupResource(context, handle, &stream_resource));
    // Remove the reference at the end of this scope.
    mutex_lock l(*stream_resource->mutex());

    OpInputList summaries_list;
    OP_REQUIRES_OK(context,
                   context->input_list(kSummariesName, &summaries_list));
    int32_t num_streams = stream_resource->num_streams();
    CHECK_EQ(static_cast<int>(num_streams), summaries_list.size());

    auto do_quantile_add_summary = [&](const int64_t begin, const int64_t end) {
      // Iterating all features.
      for (int64_t feature_idx = begin; feature_idx < end; ++feature_idx) {
        QuantileStream* stream = stream_resource->stream(feature_idx);
        if (stream->IsFinalized()) {
          VLOG(1) << "QuantileStream has already been finalized for feature"
                  << feature_idx << ".";
          continue;
        }
        const Tensor& summaries = summaries_list[feature_idx];
        const auto summary_values = summaries.matrix<float>();
        const auto& tensor_shape = summaries.shape();
        const int64_t entries_size = tensor_shape.dim_size(0);
        CHECK_EQ(tensor_shape.dim_size(1), 4);
        std::vector<QuantileSummaryEntry> summary_entries;
        summary_entries.reserve(entries_size);
        for (int64_t i = 0; i < entries_size; i++) {
          float value = summary_values(i, 0);
          float weight = summary_values(i, 1);
          float min_rank = summary_values(i, 2);
          float max_rank = summary_values(i, 3);
          QuantileSummaryEntry entry(value, weight, min_rank, max_rank);
          summary_entries.push_back(entry);
        }
        stream_resource->stream(feature_idx)->PushSummary(summary_entries);
      }
    };

    // TODO(tanzheny): comment on the magic number.
    const int64_t kCostPerUnit = 500 * num_streams;
    const DeviceBase::CpuWorkerThreads& worker_threads =
        *context->device()->tensorflow_cpu_worker_threads();
    Shard(worker_threads.num_threads, worker_threads.workers, num_streams,
          kCostPerUnit, do_quantile_add_summary);
  }


static CURLcode client_init_writer(struct Curl_easy *data,
                                   struct contenc_writer *writer)
{
  (void) data;
  return writer->downstream? CURLE_WRITE_ERROR: CURLE_OK;
}

content_encoding: return error on too many compression steps

The max allowed steps is arbitrarily set to 5.

Bug: https://curl.se/docs/CVE-2022-32206.html
CVE-2022-32206
Reported-by: Harry Sintonen
Closes #9049


static CURLcode brotli_map_error(BrotliDecoderErrorCode be)
{
  switch(be) {
  case BROTLI_DECODER_ERROR_FORMAT_EXUBERANT_NIBBLE:
  case BROTLI_DECODER_ERROR_FORMAT_EXUBERANT_META_NIBBLE:
  case BROTLI_DECODER_ERROR_FORMAT_SIMPLE_HUFFMAN_ALPHABET:
  case BROTLI_DECODER_ERROR_FORMAT_SIMPLE_HUFFMAN_SAME:
  case BROTLI_DECODER_ERROR_FORMAT_CL_SPACE:
  case BROTLI_DECODER_ERROR_FORMAT_HUFFMAN_SPACE:
  case BROTLI_DECODER_ERROR_FORMAT_CONTEXT_MAP_REPEAT:
  case BROTLI_DECODER_ERROR_FORMAT_BLOCK_LENGTH_1:
  case BROTLI_DECODER_ERROR_FORMAT_BLOCK_LENGTH_2:
  case BROTLI_DECODER_ERROR_FORMAT_TRANSFORM:
  case BROTLI_DECODER_ERROR_FORMAT_DICTIONARY:
  case BROTLI_DECODER_ERROR_FORMAT_WINDOW_BITS:
  case BROTLI_DECODER_ERROR_FORMAT_PADDING_1:
  case BROTLI_DECODER_ERROR_FORMAT_PADDING_2:
#ifdef BROTLI_DECODER_ERROR_COMPOUND_DICTIONARY
  case BROTLI_DECODER_ERROR_COMPOUND_DICTIONARY:
#endif
#ifdef BROTLI_DECODER_ERROR_DICTIONARY_NOT_SET
  case BROTLI_DECODER_ERROR_DICTIONARY_NOT_SET:
#endif
  case BROTLI_DECODER_ERROR_INVALID_ARGUMENTS:
    return CURLE_BAD_CONTENT_ENCODING;
  case BROTLI_DECODER_ERROR_ALLOC_CONTEXT_MODES:
  case BROTLI_DECODER_ERROR_ALLOC_TREE_GROUPS:
  case BROTLI_DECODER_ERROR_ALLOC_CONTEXT_MAP:
  case BROTLI_DECODER_ERROR_ALLOC_RING_BUFFER_1:
  case BROTLI_DECODER_ERROR_ALLOC_RING_BUFFER_2:
  case BROTLI_DECODER_ERROR_ALLOC_BLOCK_TYPE_TREES:
    return CURLE_OUT_OF_MEMORY;
  default:
    break;
  }
  return CURLE_WRITE_ERROR;
}


static void deflate_close_writer(struct Curl_easy *data,
                                 struct contenc_writer *writer)
{
  struct zlib_params *zp = (struct zlib_params *) &writer->params;
  z_stream *z = &zp->z;     /* zlib state structure */

  exit_zlib(data, z, &zp->zlib_init, CURLE_OK);
}


static void brotli_close_writer(struct Curl_easy *data,
                                struct contenc_writer *writer)
{
  struct brotli_params *bp = (struct brotli_params *) &writer->params;
  (void) data;

  if(bp->br) {
    BrotliDecoderDestroyInstance(bp->br);
    bp->br = NULL;
  }
}


static void zstd_close_writer(struct Curl_easy *data,
                              struct contenc_writer *writer)
{
  struct zstd_params *zp = (struct zstd_params *)&writer->params;
  (void)data;

  if(zp->decomp) {
    free(zp->decomp);
    zp->decomp = NULL;
  }
  if(zp->zds) {
    ZSTD_freeDStream(zp->zds);
    zp->zds = NULL;
  }
}


static CURLcode error_init_writer(struct Curl_easy *data,
                                  struct contenc_writer *writer)
{
  (void) data;
  return writer->downstream? CURLE_OK: CURLE_WRITE_ERROR;
}


process_zlib_error(struct Curl_easy *data, z_stream *z)
{
  if(z->msg)
    failf(data, "Error while processing content unencoding: %s",
          z->msg);
  else
    failf(data, "Error while processing content unencoding: "
          "Unknown failure within decompression software.");

  return CURLE_BAD_CONTENT_ENCODING;
}


CURLcode Curl_unencode_write(struct Curl_easy *data,
                             struct contenc_writer *writer,
                             const char *buf, size_t nbytes)
{
  (void) data;
  (void) writer;
  (void) buf;
  (void) nbytes;
  return CURLE_NOT_BUILT_IN;
}


static CURLcode deflate_unencode_write(struct Curl_easy *data,
                                       struct contenc_writer *writer,
                                       const char *buf, size_t nbytes)
{
  struct zlib_params *zp = (struct zlib_params *) &writer->params;
  z_stream *z = &zp->z;     /* zlib state structure */

  /* Set the compressed input when this function is called */
  z->next_in = (Bytef *) buf;
  z->avail_in = (uInt) nbytes;

  if(zp->zlib_init == ZLIB_EXTERNAL_TRAILER)
    return process_trailer(data, zp);

  /* Now uncompress the data */
  return inflate_stream(data, writer, ZLIB_INFLATING);
}


void Curl_unencode_cleanup(struct Curl_easy *data)
{
  struct SingleRequest *k = &data->req;
  struct contenc_writer *writer = k->writer_stack;

  while(writer) {
    k->writer_stack = writer->downstream;
    writer->handler->close_writer(data, writer);
    free(writer);
    writer = k->writer_stack;
  }
}


static CURLcode deflate_init_writer(struct Curl_easy *data,
                                    struct contenc_writer *writer)
{
  struct zlib_params *zp = (struct zlib_params *) &writer->params;
  z_stream *z = &zp->z;     /* zlib state structure */

  if(!writer->downstream)
    return CURLE_WRITE_ERROR;

  /* Initialize zlib */
  z->zalloc = (alloc_func) zalloc_cb;
  z->zfree = (free_func) zfree_cb;

  if(inflateInit(z) != Z_OK)
    return process_zlib_error(data, z);
  zp->zlib_init = ZLIB_INIT;
  return CURLE_OK;
}


static void gzip_close_writer(struct Curl_easy *data,
                              struct contenc_writer *writer)
{
  struct zlib_params *zp = (struct zlib_params *) &writer->params;
  z_stream *z = &zp->z;     /* zlib state structure */

  exit_zlib(data, z, &zp->zlib_init, CURLE_OK);
}


} check_gzip_header(unsigned char const *data, ssize_t len, ssize_t *headerlen)
{
  int method, flags;
  const ssize_t totallen = len;

  /* The shortest header is 10 bytes */
  if(len < 10)
    return GZIP_UNDERFLOW;

  if((data[0] != GZIP_MAGIC_0) || (data[1] != GZIP_MAGIC_1))
    return GZIP_BAD;

  method = data[2];
  flags = data[3];

  if(method != Z_DEFLATED || (flags & RESERVED) != 0) {
    /* Can't handle this compression method or unknown flag */
    return GZIP_BAD;
  }

  /* Skip over time, xflags, OS code and all previous bytes */
  len -= 10;
  data += 10;

  if(flags & EXTRA_FIELD) {
    ssize_t extra_len;

    if(len < 2)
      return GZIP_UNDERFLOW;

    extra_len = (data[1] << 8) | data[0];

    if(len < (extra_len + 2))
      return GZIP_UNDERFLOW;

    len -= (extra_len + 2);
    data += (extra_len + 2);
  }

  if(flags & ORIG_NAME) {
    /* Skip over NUL-terminated file name */
    while(len && *data) {
      --len;
      ++data;
    }
    if(!len || *data)
      return GZIP_UNDERFLOW;

    /* Skip over the NUL */
    --len;
    ++data;
  }

  if(flags & COMMENT) {
    /* Skip over NUL-terminated comment */
    while(len && *data) {
      --len;
      ++data;
    }
    if(!len || *data)
      return GZIP_UNDERFLOW;

    /* Skip over the NUL */
    --len;
  }

  if(flags & HEAD_CRC) {
    if(len < 2)
      return GZIP_UNDERFLOW;

    len -= 2;
  }

  *headerlen = totallen - len;
  return GZIP_OK;
}


new_unencoding_writer(struct Curl_easy *data,
                      const struct content_encoding *handler,
                      struct contenc_writer *downstream)
{
  size_t sz = offsetof(struct contenc_writer, params) + handler->paramsize;
  struct contenc_writer *writer = (struct contenc_writer *)calloc(1, sz);

  if(writer) {
    writer->handler = handler;
    writer->downstream = downstream;
    if(handler->init_writer(data, writer)) {
      free(writer);
      writer = NULL;
    }
  }

  return writer;
}


static CURLcode zstd_unencode_write(struct Curl_easy *data,
                                    struct contenc_writer *writer,
                                    const char *buf, size_t nbytes)
{
  CURLcode result = CURLE_OK;
  struct zstd_params *zp = (struct zstd_params *)&writer->params;
  ZSTD_inBuffer in;
  ZSTD_outBuffer out;
  size_t errorCode;

  if(!zp->decomp) {
    zp->decomp = malloc(DSIZ);
    if(!zp->decomp)
      return CURLE_OUT_OF_MEMORY;
  }
  in.pos = 0;
  in.src = buf;
  in.size = nbytes;

  for(;;) {
    out.pos = 0;
    out.dst = zp->decomp;
    out.size = DSIZ;

    errorCode = ZSTD_decompressStream(zp->zds, &out, &in);
    if(ZSTD_isError(errorCode)) {
      return CURLE_BAD_CONTENT_ENCODING;
    }
    if(out.pos > 0) {
      result = Curl_unencode_write(data, writer->downstream,
                                   zp->decomp, out.pos);
      if(result)
        break;
    }
    if((in.pos == nbytes) && (out.pos < out.size))
      break;
  }

  return result;
}


char *Curl_all_content_encodings(void)
{
  return strdup(CONTENT_ENCODING_DEFAULT);  /* Satisfy caller. */
}


static CURLcode client_unencode_write(struct Curl_easy *data,
                                      struct contenc_writer *writer,
                                      const char *buf, size_t nbytes)
{
  struct SingleRequest *k = &data->req;

  (void) writer;

  if(!nbytes || k->ignorebody)
    return CURLE_OK;

  return Curl_client_write(data, CLIENTWRITE_BODY, (char *) buf, nbytes);
}


static CURLcode zstd_init_writer(struct Curl_easy *data,
                                 struct contenc_writer *writer)
{
  struct zstd_params *zp = (struct zstd_params *)&writer->params;
  (void)data;

  if(!writer->downstream)
    return CURLE_WRITE_ERROR;

  zp->zds = ZSTD_createDStream();
  zp->decomp = NULL;
  return zp->zds ? CURLE_OK : CURLE_OUT_OF_MEMORY;
}


CURLcode Curl_build_unencoding_stack(struct Curl_easy *data,
                                     const char *enclist, int maybechunked)
{
  struct SingleRequest *k = &data->req;
  int counter = 0;

  do {
    const char *name;
    size_t namelen;

    /* Parse a single encoding name. */
    while(ISSPACE(*enclist) || *enclist == ',')
      enclist++;

    name = enclist;

    for(namelen = 0; *enclist && *enclist != ','; enclist++)
      if(!ISSPACE(*enclist))
        namelen = enclist - name + 1;

    /* Special case: chunked encoding is handled at the reader level. */
    if(maybechunked && namelen == 7 && strncasecompare(name, "chunked", 7)) {
      k->chunk = TRUE;             /* chunks coming our way. */
      Curl_httpchunk_init(data);   /* init our chunky engine. */
    }
    else if(namelen) {
      const struct content_encoding *encoding = find_encoding(name, namelen);
      struct contenc_writer *writer;

      if(!k->writer_stack) {
        k->writer_stack = new_unencoding_writer(data, &client_encoding, NULL);

        if(!k->writer_stack)
          return CURLE_OUT_OF_MEMORY;
      }

      if(!encoding)
        encoding = &error_encoding;  /* Defer error at stack use. */

      if(++counter >= MAX_ENCODE_STACK) {
        failf(data, "Reject response due to %u content encodings",
              counter);
        return CURLE_BAD_CONTENT_ENCODING;
      }
      /* Stack the unencoding stage. */
      writer = new_unencoding_writer(data, encoding, k->writer_stack);
      if(!writer)
        return CURLE_OUT_OF_MEMORY;
      k->writer_stack = writer;
    }
  } while(*enclist);

  return CURLE_OK;
}


static CURLcode gzip_unencode_write(struct Curl_easy *data,
                                    struct contenc_writer *writer,
                                    const char *buf, size_t nbytes)
{
  struct zlib_params *zp = (struct zlib_params *) &writer->params;
  z_stream *z = &zp->z;     /* zlib state structure */

  if(zp->zlib_init == ZLIB_INIT_GZIP) {
    /* Let zlib handle the gzip decompression entirely */
    z->next_in = (Bytef *) buf;
    z->avail_in = (uInt) nbytes;
    /* Now uncompress the data */
    return inflate_stream(data, writer, ZLIB_INIT_GZIP);
  }

#ifndef OLD_ZLIB_SUPPORT
  /* Support for old zlib versions is compiled away and we are running with
     an old version, so return an error. */
  return exit_zlib(data, z, &zp->zlib_init, CURLE_WRITE_ERROR);

#else
  /* This next mess is to get around the potential case where there isn't
   * enough data passed in to skip over the gzip header.  If that happens, we
   * malloc a block and copy what we have then wait for the next call.  If
   * there still isn't enough (this is definitely a worst-case scenario), we
   * make the block bigger, copy the next part in and keep waiting.
   *
   * This is only required with zlib versions < 1.2.0.4 as newer versions
   * can handle the gzip header themselves.
   */

  switch(zp->zlib_init) {
  /* Skip over gzip header? */
  case ZLIB_INIT:
  {
    /* Initial call state */
    ssize_t hlen;

    switch(check_gzip_header((unsigned char *) buf, nbytes, &hlen)) {
    case GZIP_OK:
      z->next_in = (Bytef *) buf + hlen;
      z->avail_in = (uInt) (nbytes - hlen);
      zp->zlib_init = ZLIB_GZIP_INFLATING; /* Inflating stream state */
      break;

    case GZIP_UNDERFLOW:
      /* We need more data so we can find the end of the gzip header.  It's
       * possible that the memory block we malloc here will never be freed if
       * the transfer abruptly aborts after this point.  Since it's unlikely
       * that circumstances will be right for this code path to be followed in
       * the first place, and it's even more unlikely for a transfer to fail
       * immediately afterwards, it should seldom be a problem.
       */
      z->avail_in = (uInt) nbytes;
      z->next_in = malloc(z->avail_in);
      if(!z->next_in) {
        return exit_zlib(data, z, &zp->zlib_init, CURLE_OUT_OF_MEMORY);
      }
      memcpy(z->next_in, buf, z->avail_in);
      zp->zlib_init = ZLIB_GZIP_HEADER;  /* Need more gzip header data state */
      /* We don't have any data to inflate yet */
      return CURLE_OK;

    case GZIP_BAD:
    default:
      return exit_zlib(data, z, &zp->zlib_init, process_zlib_error(data, z));
    }

  }
  break;

  case ZLIB_GZIP_HEADER:
  {
    /* Need more gzip header data state */
    ssize_t hlen;
    z->avail_in += (uInt) nbytes;
    z->next_in = Curl_saferealloc(z->next_in, z->avail_in);
    if(!z->next_in) {
      return exit_zlib(data, z, &zp->zlib_init, CURLE_OUT_OF_MEMORY);
    }
    /* Append the new block of data to the previous one */
    memcpy(z->next_in + z->avail_in - nbytes, buf, nbytes);

    switch(check_gzip_header(z->next_in, z->avail_in, &hlen)) {
    case GZIP_OK:
      /* This is the zlib stream data */
      free(z->next_in);
      /* Don't point into the malloced block since we just freed it */
      z->next_in = (Bytef *) buf + hlen + nbytes - z->avail_in;
      z->avail_in = (uInt) (z->avail_in - hlen);
      zp->zlib_init = ZLIB_GZIP_INFLATING;   /* Inflating stream state */
      break;

    case GZIP_UNDERFLOW:
      /* We still don't have any data to inflate! */
      return CURLE_OK;

    case GZIP_BAD:
    default:
      return exit_zlib(data, z, &zp->zlib_init, process_zlib_error(data, z));
    }

  }
  break;

  case ZLIB_EXTERNAL_TRAILER:
    z->next_in = (Bytef *) buf;
    z->avail_in = (uInt) nbytes;
    return process_trailer(data, zp);

  case ZLIB_GZIP_INFLATING:
  default:
    /* Inflating stream state */
    z->next_in = (Bytef *) buf;
    z->avail_in = (uInt) nbytes;
    break;
  }

  if(z->avail_in == 0) {
    /* We don't have any data to inflate; wait until next time */
    return CURLE_OK;
  }

  /* We've parsed the header, now uncompress the data */
  return inflate_stream(data, writer, ZLIB_GZIP_INFLATING);
#endif
}


static CURLcode brotli_unencode_write(struct Curl_easy *data,
                                      struct contenc_writer *writer,
                                      const char *buf, size_t nbytes)
{
  struct brotli_params *bp = (struct brotli_params *) &writer->params;
  const uint8_t *src = (const uint8_t *) buf;
  char *decomp;
  uint8_t *dst;
  size_t dstleft;
  CURLcode result = CURLE_OK;
  BrotliDecoderResult r = BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT;

  if(!bp->br)
    return CURLE_WRITE_ERROR;  /* Stream already ended. */

  decomp = malloc(DSIZ);
  if(!decomp)
    return CURLE_OUT_OF_MEMORY;

  while((nbytes || r == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) &&
        result == CURLE_OK) {
    dst = (uint8_t *) decomp;
    dstleft = DSIZ;
    r = BrotliDecoderDecompressStream(bp->br,
                                      &nbytes, &src, &dstleft, &dst, NULL);
    result = Curl_unencode_write(data, writer->downstream,
                                 decomp, DSIZ - dstleft);
    if(result)
      break;
    switch(r) {
    case BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT:
    case BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT:
      break;
    case BROTLI_DECODER_RESULT_SUCCESS:
      BrotliDecoderDestroyInstance(bp->br);
      bp->br = NULL;
      if(nbytes)
        result = CURLE_WRITE_ERROR;
      break;
    default:
      result = brotli_map_error(BrotliDecoderGetErrorCode(bp->br));
      break;
    }
  }
  free(decomp);
  return result;
}


exit_zlib(struct Curl_easy *data,
          z_stream *z, zlibInitState *zlib_init, CURLcode result)
{
  if(*zlib_init == ZLIB_GZIP_HEADER)
    Curl_safefree(z->next_in);

  if(*zlib_init != ZLIB_UNINIT) {
    if(inflateEnd(z) != Z_OK && result == CURLE_OK)
      result = process_zlib_error(data, z);
    *zlib_init = ZLIB_UNINIT;
  }

  return result;
}


static void identity_close_writer(struct Curl_easy *data,
                                  struct contenc_writer *writer)
{
  (void) data;
  (void) writer;
}


char *Curl_all_content_encodings(void)
{
  size_t len = 0;
  const struct content_encoding * const *cep;
  const struct content_encoding *ce;
  char *ace;

  for(cep = encodings; *cep; cep++) {
    ce = *cep;
    if(!strcasecompare(ce->name, CONTENT_ENCODING_DEFAULT))
      len += strlen(ce->name) + 2;
  }

  if(!len)
    return strdup(CONTENT_ENCODING_DEFAULT);

  ace = malloc(len);
  if(ace) {
    char *p = ace;
    for(cep = encodings; *cep; cep++) {
      ce = *cep;
      if(!strcasecompare(ce->name, CONTENT_ENCODING_DEFAULT)) {
        strcpy(p, ce->name);
        p += strlen(p);
        *p++ = ',';
        *p++ = ' ';
      }
    }
    p[-2] = '\0';
  }

  return ace;
}


static CURLcode process_trailer(struct Curl_easy *data,
                                struct zlib_params *zp)
{
  z_stream *z = &zp->z;
  CURLcode result = CURLE_OK;
  uInt len = z->avail_in < zp->trailerlen? z->avail_in: zp->trailerlen;

  /* Consume expected trailer bytes. Terminate stream if exhausted.
     Issue an error if unexpected bytes follow. */

  zp->trailerlen -= len;
  z->avail_in -= len;
  z->next_in += len;
  if(z->avail_in)
    result = CURLE_WRITE_ERROR;
  if(result || !zp->trailerlen)
    result = exit_zlib(data, z, &zp->zlib_init, result);
  else {
    /* Only occurs for gzip with zlib < 1.2.0.4 or raw deflate. */
    zp->zlib_init = ZLIB_EXTERNAL_TRAILER;
  }
  return result;
}


CURLcode Curl_unencode_write(struct Curl_easy *data,
                             struct contenc_writer *writer,
                             const char *buf, size_t nbytes)
{
  if(!nbytes)
    return CURLE_OK;
  return writer->handler->unencode_write(data, writer, buf, nbytes);
}


static void error_close_writer(struct Curl_easy *data,
                               struct contenc_writer *writer)
{
  (void) data;
  (void) writer;
}


static CURLcode identity_init_writer(struct Curl_easy *data,
                                     struct contenc_writer *writer)
{
  (void) data;
  return writer->downstream? CURLE_OK: CURLE_WRITE_ERROR;
}


zfree_cb(voidpf opaque, voidpf ptr)
{
  (void) opaque;
  free(ptr);
}


static CURLcode brotli_init_writer(struct Curl_easy *data,
                                   struct contenc_writer *writer)
{
  struct brotli_params *bp = (struct brotli_params *) &writer->params;
  (void) data;

  if(!writer->downstream)
    return CURLE_WRITE_ERROR;

  bp->br = BrotliDecoderCreateInstance(NULL, NULL, NULL);
  return bp->br? CURLE_OK: CURLE_OUT_OF_MEMORY;
}


static CURLcode gzip_init_writer(struct Curl_easy *data,
                                 struct contenc_writer *writer)
{
  struct zlib_params *zp = (struct zlib_params *) &writer->params;
  z_stream *z = &zp->z;     /* zlib state structure */

  if(!writer->downstream)
    return CURLE_WRITE_ERROR;

  /* Initialize zlib */
  z->zalloc = (alloc_func) zalloc_cb;
  z->zfree = (free_func) zfree_cb;

  if(strcmp(zlibVersion(), "1.2.0.4") >= 0) {
    /* zlib ver. >= 1.2.0.4 supports transparent gzip decompressing */
    if(inflateInit2(z, MAX_WBITS + 32) != Z_OK) {
      return process_zlib_error(data, z);
    }
    zp->zlib_init = ZLIB_INIT_GZIP; /* Transparent gzip decompress state */
  }
  else {
    /* we must parse the gzip header and trailer ourselves */
    if(inflateInit2(z, -MAX_WBITS) != Z_OK) {
      return process_zlib_error(data, z);
    }
    zp->trailerlen = 8; /* A CRC-32 and a 32-bit input size (RFC 1952, 2.2) */
    zp->zlib_init = ZLIB_INIT; /* Initial call state */
  }

  return CURLE_OK;
}


static CURLcode inflate_stream(struct Curl_easy *data,
                               struct contenc_writer *writer,
                               zlibInitState started)
{
  struct zlib_params *zp = (struct zlib_params *) &writer->params;
  z_stream *z = &zp->z;         /* zlib state structure */
  uInt nread = z->avail_in;
  Bytef *orig_in = z->next_in;
  bool done = FALSE;
  CURLcode result = CURLE_OK;   /* Curl_client_write status */
  char *decomp;                 /* Put the decompressed data here. */

  /* Check state. */
  if(zp->zlib_init != ZLIB_INIT &&
     zp->zlib_init != ZLIB_INFLATING &&
     zp->zlib_init != ZLIB_INIT_GZIP &&
     zp->zlib_init != ZLIB_GZIP_INFLATING)
    return exit_zlib(data, z, &zp->zlib_init, CURLE_WRITE_ERROR);

  /* Dynamically allocate a buffer for decompression because it's uncommonly
     large to hold on the stack */
  decomp = malloc(DSIZ);
  if(!decomp)
    return exit_zlib(data, z, &zp->zlib_init, CURLE_OUT_OF_MEMORY);

  /* because the buffer size is fixed, iteratively decompress and transfer to
     the client via downstream_write function. */
  while(!done) {
    int status;                   /* zlib status */
    done = TRUE;

    /* (re)set buffer for decompressed output for every iteration */
    z->next_out = (Bytef *) decomp;
    z->avail_out = DSIZ;

#ifdef Z_BLOCK
    /* Z_BLOCK is only available in zlib ver. >= 1.2.0.5 */
    status = inflate(z, Z_BLOCK);
#else
    /* fallback for zlib ver. < 1.2.0.5 */
    status = inflate(z, Z_SYNC_FLUSH);
#endif

    /* Flush output data if some. */
    if(z->avail_out != DSIZ) {
      if(status == Z_OK || status == Z_STREAM_END) {
        zp->zlib_init = started;      /* Data started. */
        result = Curl_unencode_write(data, writer->downstream, decomp,
                                     DSIZ - z->avail_out);
        if(result) {
          exit_zlib(data, z, &zp->zlib_init, result);
          break;
        }
      }
    }

    /* Dispatch by inflate() status. */
    switch(status) {
    case Z_OK:
      /* Always loop: there may be unflushed latched data in zlib state. */
      done = FALSE;
      break;
    case Z_BUF_ERROR:
      /* No more data to flush: just exit loop. */
      break;
    case Z_STREAM_END:
      result = process_trailer(data, zp);
      break;
    case Z_DATA_ERROR:
      /* some servers seem to not generate zlib headers, so this is an attempt
         to fix and continue anyway */
      if(zp->zlib_init == ZLIB_INIT) {
        /* Do not use inflateReset2(): only available since zlib 1.2.3.4. */
        (void) inflateEnd(z);     /* don't care about the return code */
        if(inflateInit2(z, -MAX_WBITS) == Z_OK) {
          z->next_in = orig_in;
          z->avail_in = nread;
          zp->zlib_init = ZLIB_INFLATING;
          zp->trailerlen = 4; /* Tolerate up to 4 unknown trailer bytes. */
          done = FALSE;
          break;
        }
        zp->zlib_init = ZLIB_UNINIT;    /* inflateEnd() already called. */
      }
      result = exit_zlib(data, z, &zp->zlib_init, process_zlib_error(data, z));
      break;
    default:
      result = exit_zlib(data, z, &zp->zlib_init, process_zlib_error(data, z));
      break;
    }
  }
  free(decomp);

  /* We're about to leave this call so the `nread' data bytes won't be seen
     again. If we are in a state that would wrongly allow restart in raw mode
     at the next call, assume output has already started. */
  if(nread && zp->zlib_init == ZLIB_INIT)
    zp->zlib_init = started;      /* Cannot restart anymore. */

  return result;
}


static void client_close_writer(struct Curl_easy *data,
                                struct contenc_writer *writer)
{
  (void) data;
  (void) writer;
}


zalloc_cb(voidpf opaque, unsigned int items, unsigned int size)
{
  (void) opaque;
  /* not a typo, keep it calloc() */
  return (voidpf) calloc(items, size);
}


static const struct content_encoding *find_encoding(const char *name,
                                                    size_t len)
{
  const struct content_encoding * const *cep;

  for(cep = encodings; *cep; cep++) {
    const struct content_encoding *ce = *cep;
    if((strncasecompare(name, ce->name, len) && !ce->name[len]) ||
       (ce->alias && strncasecompare(name, ce->alias, len) && !ce->alias[len]))
      return ce;
  }
  return NULL;
}


void Curl_unencode_cleanup(struct Curl_easy *data)
{
  (void) data;
}


static CURLcode error_unencode_write(struct Curl_easy *data,
                                     struct contenc_writer *writer,
                                     const char *buf, size_t nbytes)
{
  char *all = Curl_all_content_encodings();

  (void) writer;
  (void) buf;
  (void) nbytes;

  if(!all)
    return CURLE_OUT_OF_MEMORY;
  failf(data, "Unrecognized content encoding type. "
        "libcurl understands %s content encodings.", all);
  free(all);
  return CURLE_BAD_CONTENT_ENCODING;
}


static CURLcode identity_unencode_write(struct Curl_easy *data,
                                        struct contenc_writer *writer,
                                        const char *buf, size_t nbytes)
{
  return Curl_unencode_write(data, writer->downstream, buf, nbytes);
}


void fmtutil_render_uuid(deark *c, const u8 *uuid, char *s, size_t s_len)
{
	de_snprintf(s, s_len, "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x",
		uuid[0], uuid[1], uuid[2], uuid[3], uuid[4], uuid[5], uuid[6], uuid[7],
		uuid[8], uuid[9], uuid[10], uuid[11], uuid[12], uuid[13], uuid[14], uuid[15]);
}

pict,macrsrc: Fixed a bug that could cause division by 0

Found by F. Çelik.


void fmtutil_handle_iptc(deark *c, dbuf *f, i64 pos, i64 len,
	unsigned int flags)
{
	int should_decode;
	int should_extract;
	int user_opt;
	int extract_fmt = 1; // 0=raw, 1=TIFF-wrapped

	if(len<1) return;

	user_opt = de_get_ext_option_bool(c, "extractiptc", -1);

	if(user_opt==1 || (c->extract_level>=2 && user_opt!=0)) {
		should_decode = 0;
		should_extract = 1;
		if(flags&0x2) {
			// Avoid "extracting" in a way that would just recreate the exact same file.
			extract_fmt = 0;
		}
	}
	else {
		should_decode = 1;
		should_extract = 0;
	}

	if(should_decode) {
		de_run_module_by_id_on_slice(c, "iptc", NULL, f, pos, len);
	}

	if(should_extract && extract_fmt==0) {
		dbuf_create_file_from_slice(f, pos, len, "iptc", NULL, DE_CREATEFLAG_IS_AUX);
	}
	else if(should_extract && extract_fmt==1) {
		wrap_in_tiff(c, f, pos, len, "Deark extracted IPTC", 33723, "iptctiff",
			DE_CREATEFLAG_IS_AUX);
	}
}


static void do_iff_anno(deark *c, struct de_iffctx *ictx, i64 pos, i64 len)
{
	i64 foundpos;

	if(len<1) return;

	// Some ANNO chunks seem to be padded with one or more NUL bytes. Probably
	// best not to save them.
	if(dbuf_search_byte(ictx->f, 0x00, pos, len, &foundpos)) {
		len = foundpos - pos;
	}
	if(len<1) return;
	if(c->extract_level>=2) {
		dbuf_create_file_from_slice(ictx->f, pos, len, "anno.txt", NULL, DE_CREATEFLAG_IS_AUX);
	}
	else {
		de_ucstring *s = NULL;
		s = ucstring_create(c);
		dbuf_read_to_ucstring_n(ictx->f, pos, len, DE_DBG_MAX_STRLEN, s, 0, ictx->input_encoding);
		de_dbg(c, "annotation: \"%s\"", ucstring_getpsz(s));
		ucstring_destroy(s);
	}
}


int fmtutil_is_standard_iff_chunk(deark *c, struct de_iffctx *ictx,
	u32 ct)
{
	switch(ct) {
	case CODE__c_:
	case CODE_ANNO:
	case CODE_AUTH:
	case CODE_NAME:
	case CODE_TEXT:
		return 1;
	}
	return 0;
}


const char *fmtutil_get_windows_charset_name(u8 cs)
{
	struct csname_struct { u8 id; const char *name; };
	static const struct csname_struct csname_arr[] = {
		{0x00, "ANSI"},
		{0x01, "default"},
		{0x02, "symbol"},
		{0x4d, "Mac"},
		{0x80, "Shift-JIS"},
		{0x81, "Hangul"},
		{0x82, "Johab"},
		{0x86, "GB2312"},
		{0x88, "BIG5"},
		{0xa1, "Greek"},
		{0xa2, "Turkish"},
		{0xa3, "Vietnamese"},
		{0xb1, "Hebrew"},
		{0xb2, "Arabic"},
		{0xba, "Baltic"},
		{0xcc, "Russian"},
		{0xde, "Thai"},
		{0xee, "Eastern Europe"},
		{0xff, "OEM"}
	};
	size_t i;

	for(i=0; i<DE_ARRAYCOUNT(csname_arr); i++) {
		if(cs==csname_arr[i].id) return csname_arr[i].name;
	}
	return "?";
}


void fmtutil_handle_id3(deark *c, dbuf *f, struct de_id3info *id3i,
	unsigned int flags)
{
	i64 id3v1pos = 0;
	int look_for_id3v1;

	de_zeromem(id3i, sizeof(struct de_id3info));
	id3i->main_start = 0;
	id3i->main_end = f->len;

	id3i->has_id3v2 = !dbuf_memcmp(f, 0, "ID3", 3);
	if(id3i->has_id3v2) {
		de_module_params id3v2mparams;

		de_dbg(c, "ID3v2 data at %d", 0);
		de_dbg_indent(c, 1);
		de_zeromem(&id3v2mparams, sizeof(de_module_params));
		id3v2mparams.in_params.codes = "I";
		de_run_module_by_id_on_slice(c, "id3", &id3v2mparams, f, 0, f->len);
		de_dbg_indent(c, -1);
		id3i->main_start += id3v2mparams.out_params.int64_1;
	}

	look_for_id3v1 = 1;
	if(look_for_id3v1) {
		id3v1pos = f->len-128;
		if(!dbuf_memcmp(f, id3v1pos, "TAG", 3)) {
			id3i->has_id3v1 = 1;
		}
	}

	if(id3i->has_id3v1) {
		de_module_params id3v1mparams;

		de_dbg(c, "ID3v1 data at %"I64_FMT, id3v1pos);
		de_dbg_indent(c, 1);
		de_zeromem(&id3v1mparams, sizeof(de_module_params));
		id3v1mparams.in_params.codes = "1";
		de_run_module_by_id_on_slice(c, "id3", &id3v1mparams, f, id3v1pos, 128);
		de_dbg_indent(c, -1);
		id3i->main_end = id3v1pos;
	}
}


static void dbg_timestamp(deark *c, struct de_timestamp *ts, const char *name)
{
	char timestamp_buf[64];

	de_timestamp_to_string(ts, timestamp_buf, sizeof(timestamp_buf), 0);
	de_dbg(c, "%s: %s", name, timestamp_buf);
}


const char *fmtutil_get_windows_cb_data_type_name(unsigned int ty)
{
	const char *name = "?";

	switch(ty) {
	case 1: name="CF_TEXT"; break;
	case 2: name="CF_BITMAP"; break;
	case 3: name="CF_METAFILEPICT"; break;
	case 6: name="CF_TIFF"; break;
	case 7: name="CF_OEMTEXT"; break;
	case 8: name="CF_DIB"; break;
	case 11: name="CF_RIFF"; break;
	case 12: name="CF_WAVE"; break;
	case 13: name="CF_UNICODETEXT"; break;
	case 14: name="CF_ENHMETAFILE"; break;
	case 17: name="CF_DIBV5"; break;
	}
	return name;
}


void fmtutil_riscos_read_attribs_field(deark *c, dbuf *f, struct de_riscos_file_attrs *rfa,
	i64 pos, unsigned int flags)
{
	rfa->attribs = (u32)dbuf_getu32le(f, pos);
	de_dbg(c, "attribs: 0x%08x", (unsigned int)rfa->attribs);
	de_dbg_indent(c, 1);
	rfa->crc_from_attribs = rfa->attribs>>16;
	if(flags & DE_RISCOS_FLAG_HAS_CRC) {
		de_dbg(c, "crc (reported): 0x%04x", (unsigned int)rfa->crc_from_attribs);
	}
	if(flags & DE_RISCOS_FLAG_HAS_LZWMAXBITS) {
		rfa->lzwmaxbits = (unsigned int)((rfa->attribs&0xff00)>>8);
		de_dbg(c, "lzw maxbits: %u", rfa->lzwmaxbits);
	}
	de_dbg_indent(c, -1);
}


void fmtutil_atari_help_palbits(deark *c)
{
	de_msg(c, "-opt atari:palbits=<9|12|15> : Numer of significant bits "
		"per palette color");
}


static int decode_atari_image_paletted(deark *c, struct atari_img_decode_data *adata)
{
	i64 i, j;
	i64 plane;
	i64 rowspan;
	u8 b;
	u32 v;
	i64 planespan;
	i64 ncolors;

	planespan = 2*((adata->w+15)/16);
	rowspan = planespan*adata->bpp;
	if(adata->ncolors>0)
		ncolors = adata->ncolors;
	else
		ncolors = ((i64)1)<<adata->bpp;

	for(j=0; j<adata->h; j++) {
		for(i=0; i<adata->w; i++) {
			v = 0;

			for(plane=0; plane<adata->bpp; plane++) {
				if(adata->was_compressed==0) {
					// TODO: Simplify this.
					if(adata->bpp==1) {
						b = de_get_bits_symbol(adata->unc_pixels, 1, j*rowspan, i);
					}
					else if(adata->bpp==2) {
						b = de_get_bits_symbol(adata->unc_pixels, 1,
							j*rowspan + 2*plane + (i/16)*2, i);
					}
					else if(adata->bpp==4) {
						b = de_get_bits_symbol(adata->unc_pixels, 1,
							j*rowspan + 2*plane + (i/2-(i/2)%16)+8*((i%32)/16), i%16);
					}
					else if(adata->bpp==8) {
						b = de_get_bits_symbol(adata->unc_pixels, 1,
							j*rowspan + 2*plane + (i-i%16), i%16);
					}
					else {
						b = 0;
					}
				}
				else {
					b = de_get_bits_symbol(adata->unc_pixels, 1, j*rowspan + plane*planespan, i);
				}
				if(b) v |= 1<<plane;
			}

			if(adata->is_spectrum512) {
				v = spectrum512_FindIndex(i, v);
				if(j>0) {
					v += (unsigned int)(48*(j));
				}
			}
			if(v>=(unsigned int)ncolors) v=(unsigned int)(ncolors-1);

			de_bitmap_setpixel_rgb(adata->img, i, j, adata->pal[v]);
		}
	}
	return 1;
}


void fmtutil_handle_exif(deark *c, i64 pos, i64 len)
{
	fmtutil_handle_exif2(c, pos, len, NULL, NULL, NULL);
}


void fmtutil_read_boxes_format(deark *c, struct de_boxesctx *bctx)
{
	if(!bctx->f || !bctx->handle_box_fn) return; // Internal error
	if(bctx->curbox) return; // Internal error
	do_box_sequence(c, bctx, 0, bctx->f->len, -1, 0);
}


void fmtutil_read_atari_palette(deark *c, dbuf *f, i64 pos,
	de_color *dstpal, i64 ncolors_to_read, i64 ncolors_used, unsigned int flags)
{
	i64 i;
	unsigned int n;
	int pal_bits = 0; // 9, 12, or 15. 0 = not yet determined
	u8 cr, cg, cb;
	u8 cr1, cg1, cb1;
	char cbuf[32];
	char tmps[64];
	const char *s;

	s = de_get_ext_option(c, "atari:palbits");
	if(s) {
		pal_bits = de_atoi(s);
	}

	if(pal_bits==0 && (flags&DE_FLAG_ATARI_15BIT_PAL)) {
		pal_bits = 15;
	}

	if(pal_bits==0) {
		// Pre-scan the palette, and try to guess whether Atari STE-style 12-bit
		// colors are used, instead of the usual 9-bit colors.
		// I don't know the best way to do this. Sometimes the 4th bit in each
		// nibble is used for extra color detail, and sometimes it just seems to
		// contain garbage. Maybe the logic should also depend on the file
		// format, or the number of colors.
		int bit_3_used = 0;
		int nibble_3_used = 0;

		for(i=0; i<ncolors_to_read; i++) {
			n = (unsigned int)dbuf_getu16be(f, pos + i*2);
			if(n&0xf000) {
				nibble_3_used = 1;
			}
			if(n&0x0888) {
				bit_3_used = 1;
			}
		}

		if(bit_3_used && !nibble_3_used) {
			de_dbg(c, "12-bit palette colors detected");
			pal_bits = 12;
		}
	}

	if(pal_bits<12) { // Default to 9 if <12
		pal_bits = 9;
	}
	else if(pal_bits<15) {
		pal_bits = 12;
	}
	else {
		pal_bits = 15;
	}

	for(i=0; i<ncolors_to_read; i++) {
		n = (unsigned int)dbuf_getu16be(f, pos + 2*i);

		if(pal_bits==15) {
			cr1 = (u8)((n>>6)&0x1c);
			if(n&0x0800) cr1+=2;
			if(n&0x8000) cr1++;
			cg1 = (u8)((n>>2)&0x1c);
			if(n&0x0080) cg1+=2;
			if(n&0x4000) cg1++;
			cb1 = (u8)((n<<2)&0x1c);
			if(n&0x0008) cb1+=2;
			if(n&0x2000) cb1++;
			cr = de_scale_n_to_255(31, cr1);
			cg = de_scale_n_to_255(31, cg1);
			cb = de_scale_n_to_255(31, cb1);
			de_snprintf(cbuf, sizeof(cbuf), "%2d,%2d,%2d",
				(int)cr1, (int)cg1, (int)cb1);
		}
		else if(pal_bits==12) {
			cr1 = (u8)((n>>7)&0x0e);
			if(n&0x800) cr1++;
			cg1 = (u8)((n>>3)&0x0e);
			if(n&0x080) cg1++;
			cb1 = (u8)((n<<1)&0x0e);
			if(n&0x008) cb1++;
			cr = scale_15_to_255(cr1);
			cg = scale_15_to_255(cg1);
			cb = scale_15_to_255(cb1);
			de_snprintf(cbuf, sizeof(cbuf), "%2d,%2d,%2d",
				(int)cr1, (int)cg1, (int)cb1);
		}
		else {
			cr1 = (u8)((n>>8)&0x07);
			cg1 = (u8)((n>>4)&0x07);
			cb1 = (u8)(n&0x07);
			cr = scale_7_to_255(cr1);
			cg = scale_7_to_255(cg1);
			cb = scale_7_to_255(cb1);
			de_snprintf(cbuf, sizeof(cbuf), "%d,%d,%d",
				(int)cr1, (int)cg1, (int)cb1);
		}

		dstpal[i] = DE_MAKE_RGB(cr, cg, cb);
		de_snprintf(tmps, sizeof(tmps), "0x%04x (%s) "DE_CHAR_RIGHTARROW" ", n, cbuf);
		de_dbg_pal_entry2(c, i, dstpal[i], tmps, NULL,
			(i>=ncolors_used)?" [unused]":"");
	}
}


static int decode_atari_image_16(deark *c, struct atari_img_decode_data *adata)
{
	i64 i, j;
	i64 rowspan;
	u32 v;

	rowspan = adata->w * 2;

	for(j=0; j<adata->h; j++) {
		for(i=0; i<adata->w; i++) {
			v = (u32)dbuf_getu16be(adata->unc_pixels, j*rowspan + 2*i);
			v = de_rgb565_to_888(v);
			de_bitmap_setpixel_rgb(adata->img, i, j,v);
		}
	}
	return 1;
}


void fmtutil_default_iff_chunk_identify(deark *c, struct de_iffctx *ictx)
{
	const char *name = NULL;

	switch(ictx->chunkctx->chunk4cc.id) {
	case CODE__c_ : name="copyright"; break;
	case CODE_ANNO: name="annotation"; break;
	case CODE_AUTH: name="author"; break;
	}

	if(name) {
		ictx->chunkctx->chunk_name = name;
	}
}


int fmtutil_find_zip_eocd(deark *c, dbuf *f, i64 *foundpos)
{
	u32 sig;
	u8 *buf = NULL;
	int retval = 0;
	i64 buf_offset;
	i64 buf_size;
	i64 i;

	*foundpos = 0;
	if(f->len < 22) goto done;

	// End-of-central-dir record usually starts 22 bytes from EOF. Try that first.
	sig = (u32)dbuf_getu32le(f, f->len - 22);
	if(sig == 0x06054b50U) {
		*foundpos = f->len - 22;
		retval = 1;
		goto done;
	}

	// Search for the signature.
	// The end-of-central-directory record could theoretically appear anywhere
	// in the file. We'll follow Info-Zip/UnZip's lead and search the last 66000
	// bytes.
#define MAX_ZIP_EOCD_SEARCH 66000
	buf_size = f->len;
	if(buf_size > MAX_ZIP_EOCD_SEARCH) buf_size = MAX_ZIP_EOCD_SEARCH;

	buf = de_malloc(c, buf_size);
	buf_offset = f->len - buf_size;
	dbuf_read(f, buf, buf_offset, buf_size);

	for(i=buf_size-22; i>=0; i--) {
		if(buf[i]=='P' && buf[i+1]=='K' && buf[i+2]==5 && buf[i+3]==6) {
			*foundpos = buf_offset + i;
			retval = 1;
			goto done;
		}
	}

done:
	de_free(c, buf);
	return retval;
}


i64 fmtutil_hlp_get_cus_p(dbuf *f, i64 *ppos)
{
	i64 x1, x2;

	x1 = (i64)dbuf_getbyte_p(f, ppos);
	if(x1%2 == 0) {
		// If it's even, divide by two.
		return x1>>1;
	}
	// If it's odd, divide by two, and add 128 times the value of
	// the next byte.
	x2 = (i64)dbuf_getbyte_p(f, ppos);
	return (x1>>1) | (x2<<7);
}


i64 fmtutil_hlp_get_csl_p(dbuf *f, i64 *ppos)
{
	i64 x1, x2;

	x1 = dbuf_getu16le_p(f, ppos);

	if(x1%2 == 0) {
		// If it's even, divide by two, and subtract 16384
		return (x1>>1) - 16384;
	}
	// If it's odd, divide by two, add 32768 times the value of
	// the next two bytes, and subtract 67108864.
	x1 >>= 1;
	x2 = dbuf_getu16le_p(f, ppos);
	x1 += x2*32768;
	x1 -= 67108864;
	return x1;
}


void fmtutil_handle_photoshop_rsrc(deark *c, dbuf *f, i64 pos, i64 len,
	unsigned int flags)
{
	fmtutil_handle_photoshop_rsrc2(c, f, pos, len, flags, NULL);
}


void fmtutil_generate_bmpfileheader(deark *c, dbuf *outf, const struct de_bmpinfo *bi,
	i64 file_size_override)
{
	i64 file_size_to_write;

	dbuf_write(outf, (const u8*)"BM", 2);

	if(file_size_override)
		file_size_to_write = file_size_override;
	else
		file_size_to_write = 14 + bi->total_size;
	dbuf_writeu32le(outf, file_size_to_write);

	dbuf_write_zeroes(outf, 4);
	dbuf_writeu32le(outf, 14 + bi->size_of_headers_and_pal);
}


static void do_iff_text_chunk(deark *c, struct de_iffctx *ictx, i64 dpos, i64 dlen,
	const char *name)
{
	de_ucstring *s = NULL;

	if(dlen<1) return;
	s = ucstring_create(c);
	dbuf_read_to_ucstring_n(ictx->f,
		dpos, dlen, DE_DBG_MAX_STRLEN,
		s, DE_CONVFLAG_STOP_AT_NUL, ictx->input_encoding);
	de_dbg(c, "%s: \"%s\"", name, ucstring_getpsz(s));
	ucstring_destroy(s);
}


static u8 scale_15_to_255(u8 x)
{
	return x*17;
}


int fmtutil_atari_decode_image(deark *c, struct atari_img_decode_data *adata)
{
	switch(adata->bpp) {
	case 16:
		return decode_atari_image_16(c, adata);
	case 8: case 4: case 2: case 1:
		return decode_atari_image_paletted(c, adata);
	}

	de_err(c, "Unsupported bits/pixel (%d)", (int)adata->bpp);
	return 0;
}


void fmtutil_handle_photoshop_rsrc2(deark *c, dbuf *f, i64 pos, i64 len,
	unsigned int flags, struct de_module_out_params *oparams)
{
	int should_decode;
	int should_extract;
	int extract_fmt = 1; // 0=raw, 1=TIFF-wrapped

	if(flags&0x1) {
		should_decode = 0;
		should_extract = 1;
	}
	else if(de_get_ext_option_bool(c, "extract8bim", 0)) {
		should_extract = 1;
		should_decode = 0;
		if(flags&0x2) {
			// Avoid "extracting" in a way that would just recreate the exact same file.
			extract_fmt = 0;
		}
	}
	else {
		should_decode = 1;
		should_extract = 0;
	}

	if(should_decode) {
		de_module_params *mparams = NULL;

		mparams = de_malloc(c, sizeof(de_module_params));
		mparams->in_params.codes = "R";
		if(oparams) {
			// Since mparams->out_params is an embedded struct, not a pointer,
			// we have to copy oparam's fields to and from it.
			mparams->out_params = *oparams; // struct copy
		}
		de_run_module_by_id_on_slice(c, "psd", mparams, f, pos, len);
		if(oparams) {
			*oparams = mparams->out_params; // struct copy
		}
		de_free(c, mparams);
	}

	if(should_extract && extract_fmt==0) {
		dbuf_create_file_from_slice(f, pos, len, "8bim", NULL, DE_CREATEFLAG_IS_AUX);
	}
	else if(should_extract && extract_fmt==1) {
		wrap_in_tiff(c, f, pos, len, "Deark extracted 8BIM", 34377, "8bimtiff",
			DE_CREATEFLAG_IS_AUX);
	}
}


void fmtutil_read_iff_format(deark *c, struct de_iffctx *ictx,
	i64 pos, i64 len)
{
	if(!ictx->f || !ictx->handle_chunk_fn) return; // Internal error

	ictx->level = 0;
	fourcc_clear(&ictx->main_fmt4cc);
	fourcc_clear(&ictx->main_contentstype4cc);
	fourcc_clear(&ictx->curr_container_fmt4cc);
	fourcc_clear(&ictx->curr_container_contentstype4cc);
	if(ictx->alignment==0) {
		ictx->alignment = 2;
	}
	if(ictx->sizeof_len==0) {
		ictx->sizeof_len = 4;
	}

	if(ictx->input_encoding==DE_ENCODING_UNKNOWN) {
		ictx->input_encoding = DE_ENCODING_ASCII;
	}

	do_iff_chunk_sequence(c, ictx, pos, len, 0);
}


int fmtutil_macbitmap_read_colortable(deark *c, dbuf *f,
	struct fmtutil_macbitmap_info *bi, i64 pos, i64 *bytes_used)
{
	i64 ct_id;
	u32 ct_flags;
	i64 ct_size;
	i64 k, z;
	u32 s[4];
	u8 cr, cg, cb;
	u32 clr;
	char tmps[64];

	*bytes_used = 0;
	de_dbg(c, "color table at %"I64_FMT, pos);
	de_dbg_indent(c, 1);

	ct_id = dbuf_getu32be(f, pos);
	ct_flags = (u32)dbuf_getu16be(f, pos+4); // a.k.a. transIndex
	ct_size = dbuf_getu16be(f, pos+6);
	bi->num_pal_entries = ct_size+1;
	de_dbg(c, "color table id=0x%08x, flags=0x%04x, colors=%d", (unsigned int)ct_id,
		(unsigned int)ct_flags, (int)bi->num_pal_entries);

	for(k=0; k<bi->num_pal_entries; k++) {
		for(z=0; z<4; z++) {
			s[z] = (u32)dbuf_getu16be(f, pos+8+8*k+2*z);
		}
		cr = (u8)(s[1]>>8);
		cg = (u8)(s[2]>>8);
		cb = (u8)(s[3]>>8);
		clr = DE_MAKE_RGB(cr,cg,cb);
		de_snprintf(tmps, sizeof(tmps), "(%5d,%5d,%5d,idx=%3d) "DE_CHAR_RIGHTARROW" ",
			(int)s[1], (int)s[2], (int)s[3], (int)s[0]);
		de_dbg_pal_entry2(c, k, clr, tmps, NULL, NULL);

		// Some files don't have the palette indices set. Most PICT decoders ignore
		// the indices if the "device" flag of ct_flags is set, and that seems to
		// work (though it's not clearly documented).
		if(ct_flags & 0x8000U) {
			s[0] = (u32)k;
		}

		if(s[0]<=255) {
			bi->pal[s[0]] = clr;
		}
	}

	de_dbg_indent(c, -1);
	*bytes_used = 8 + 8*bi->num_pal_entries;
	return 1;
}


int fmtutil_default_box_handler(deark *c, struct de_boxesctx *bctx)
{
	struct de_boxdata *curbox = bctx->curbox;

	if(curbox->is_uuid) {
		if(!de_memcmp(curbox->uuid, "\xb1\x4b\xf8\xbd\x08\x3d\x4b\x43\xa5\xae\x8c\xd7\xd5\xa6\xce\x03", 16)) {
			de_dbg(c, "GeoTIFF data at %"I64_FMT", len=%"I64_FMT, curbox->payload_pos, curbox->payload_len);
			dbuf_create_file_from_slice(bctx->f, curbox->payload_pos, curbox->payload_len, "geo.tif", NULL, DE_CREATEFLAG_IS_AUX);
		}
		else if(!de_memcmp(curbox->uuid, "\xbe\x7a\xcf\xcb\x97\xa9\x42\xe8\x9c\x71\x99\x94\x91\xe3\xaf\xac", 16)) {
			de_dbg(c, "XMP data at %"I64_FMT", len=%"I64_FMT, curbox->payload_pos, curbox->payload_len);
			dbuf_create_file_from_slice(bctx->f, curbox->payload_pos, curbox->payload_len, "xmp", NULL, DE_CREATEFLAG_IS_AUX);
		}
		else if(!de_memcmp(curbox->uuid, "\x2c\x4c\x01\x00\x85\x04\x40\xb9\xa0\x3e\x56\x21\x48\xd6\xdf\xeb", 16)) {
			de_dbg(c, "Photoshop resources at %"I64_FMT", len=%"I64_FMT, curbox->payload_pos, curbox->payload_len);
			de_dbg_indent(c, 1);
			fmtutil_handle_photoshop_rsrc(c, bctx->f, curbox->payload_pos, curbox->payload_len, 0x0);
			de_dbg_indent(c, -1);
		}
		else if(!de_memcmp(curbox->uuid, "\x05\x37\xcd\xab\x9d\x0c\x44\x31\xa7\x2a\xfa\x56\x1f\x2a\x11\x3e", 16) ||
			!de_memcmp(curbox->uuid, "JpgTiffExif->JP2", 16))
		{
			de_dbg(c, "Exif data at %"I64_FMT", len=%"I64_FMT, curbox->payload_pos, curbox->payload_len);
			de_dbg_indent(c, 1);
			fmtutil_handle_exif(c, curbox->payload_pos, curbox->payload_len);
			de_dbg_indent(c, -1);
		}
	}
	return 1;
}


static int do_iff_chunk(deark *c, struct de_iffctx *ictx, i64 pos, i64 bytes_avail,
	int level, i64 *pbytes_consumed)
{
	int ret;
	i64 chunk_dlen_raw;
	i64 chunk_dlen_padded;
	i64 data_bytes_avail;
	i64 hdrsize;
	struct de_iffchunkctx chunkctx;
	int saved_indent_level;
	int retval = 0;
	char name_str[80];

	de_zeromem(&chunkctx, sizeof(struct de_iffchunkctx));

	de_dbg_indent_save(c, &saved_indent_level);

	hdrsize = 4+ictx->sizeof_len;
	if(bytes_avail<hdrsize) {
		de_warn(c, "Ignoring %"I64_FMT" bytes at %"I64_FMT"; too small "
			"to be a chunk", bytes_avail, pos);
		goto done;
	}
	data_bytes_avail = bytes_avail-hdrsize;

	dbuf_read_fourcc(ictx->f, pos, &chunkctx.chunk4cc, 4,
		ictx->reversed_4cc ? DE_4CCFLAG_REVERSED : 0x0);
	if(chunkctx.chunk4cc.id==0 && level==0) {
		de_warn(c, "Chunk ID not found at %"I64_FMT"; assuming the data ends "
			"here", pos);
		goto done;
	}

	if(ictx->sizeof_len==2) {
		chunk_dlen_raw = dbuf_getu16x(ictx->f, pos+4, ictx->is_le);
	}
	else {
		chunk_dlen_raw = dbuf_getu32x(ictx->f, pos+4, ictx->is_le);
	}
	chunkctx.dlen = chunk_dlen_raw;
	chunkctx.dpos = pos+hdrsize;

	// TODO: Setting these fields (prior to the identify function) is enough
	// for now, but we should also set the other fields here if we can.
	ictx->level = level;
	ictx->chunkctx = &chunkctx;

	if(ictx->preprocess_chunk_fn) {
		ictx->preprocess_chunk_fn(c, ictx);
	}

	if(chunkctx.chunk_name) {
		de_snprintf(name_str, sizeof(name_str), " (%s)", chunkctx.chunk_name);
	}
	else {
		name_str[0] = '\0';
	}

	de_dbg(c, "chunk '%s'%s at %"I64_FMT", dpos=%"I64_FMT", dlen=%"I64_FMT,
		chunkctx.chunk4cc.id_dbgstr, name_str, pos,
		chunkctx.dpos, chunkctx.dlen);
	de_dbg_indent(c, 1);

	if(chunkctx.dlen > data_bytes_avail) {
		int should_warn = 1;

		if(chunkctx.chunk4cc.id==CODE_RIFF && pos==0 && bytes_avail==ictx->f->len) {
			// Hack:
			// This apparent error, in which the RIFF chunk's length field gives the
			// length of the entire file, is too common (particularly in .ani files)
			// to warn about.
			should_warn = 0;
		}

		if(should_warn) {
			de_warn(c, "Invalid oversized chunk, or unexpected end of file "
				"(chunk at %d ends at %" I64_FMT ", "
				"parent ends at %" I64_FMT ")",
				(int)pos, chunkctx.dlen+chunkctx.dpos, pos+bytes_avail);
		}

		chunkctx.dlen = data_bytes_avail; // Try to continue
		de_dbg(c, "adjusting chunk data len to %"I64_FMT, chunkctx.dlen);
	}

	chunk_dlen_padded = de_pad_to_n(chunkctx.dlen, ictx->alignment);
	*pbytes_consumed = hdrsize + chunk_dlen_padded;

	// We've set *pbytes_consumed, so we can return "success"
	retval = 1;

	// Set ictx fields, prior to calling the handler
	chunkctx.pos = pos;
	chunkctx.len = bytes_avail;
	ictx->handled = 0;
	ictx->is_std_container = 0;
	ictx->is_raw_container = 0;

	ret = ictx->handle_chunk_fn(c, ictx);
	if(!ret) {
		retval = 0;
		goto done;
	}

	if(ictx->is_std_container || ictx->is_raw_container) {
		i64 contents_dpos, contents_dlen;

		ictx->chunkctx = NULL;
		ictx->curr_container_fmt4cc = chunkctx.chunk4cc;
		fourcc_clear(&ictx->curr_container_contentstype4cc);

		if(ictx->is_std_container) {
			contents_dpos = chunkctx.dpos+4;
			contents_dlen = chunkctx.dlen-4;

			// First 4 bytes of payload are the "contents type" or "FORM type"
			dbuf_read_fourcc(ictx->f, chunkctx.dpos, &ictx->curr_container_contentstype4cc, 4,
				ictx->reversed_4cc ? DE_4CCFLAG_REVERSED : 0);

			if(level==0) {
				ictx->main_fmt4cc = ictx->curr_container_fmt4cc;
				ictx->main_contentstype4cc = ictx->curr_container_contentstype4cc; // struct copy
			}
			de_dbg(c, "contents type: '%s'", ictx->curr_container_contentstype4cc.id_dbgstr);

			if(ictx->on_std_container_start_fn) {
				// Call only for standard-format containers.
				ret = ictx->on_std_container_start_fn(c, ictx);
				if(!ret) goto done;
			}
		}
		else { // ictx->is_raw_container
			contents_dpos = chunkctx.dpos;
			contents_dlen = chunkctx.dlen;
		}

		ret = do_iff_chunk_sequence(c, ictx, contents_dpos, contents_dlen, level+1);
		if(!ret) {
			retval = 0;
			goto done;
		}

		if(ictx->on_container_end_fn) {
			// Call for all containers (not just standard-format containers).

			// TODO: Decide exactly what ictx->* fields to set here.
			ictx->level = level;

			ictx->chunkctx = NULL;
			ret = ictx->on_container_end_fn(c, ictx);
			if(!ret) {
				retval = 0;
				goto done;
			}
		}
	}
	else if(!ictx->handled) {
		de_fmtutil_default_iff_chunk_handler(c, ictx);
	}

done:
	fourcc_clear(&ictx->curr_container_fmt4cc);
	fourcc_clear(&ictx->curr_container_contentstype4cc);

	de_dbg_indent_restore(c, saved_indent_level);
	return retval;
}


int fmtutil_get_bmpinfo(deark *c, dbuf *f, struct de_bmpinfo *bi, i64 pos,
	i64 len, unsigned int flags)
{
	i64 fhs; // file header size
	i64 bmih_pos;
	struct de_fourcc cmpr4cc;
	char cmprname_dbgstr[80];

	de_zeromem(bi, sizeof(struct de_bmpinfo));
	de_zeromem(&cmpr4cc, sizeof(struct de_fourcc));

	fhs = (flags & DE_BMPINFO_HAS_FILEHEADER) ? 14 : 0;

	if(fhs+len < 16) return 0;

	if(fhs) {
		if(flags & DE_BMPINFO_HAS_HOTSPOT) {
			bi->hotspot_x = (int)dbuf_getu16le(f, pos+6);
			bi->hotspot_y = (int)dbuf_getu16le(f, pos+8);
			de_dbg(c, "hotspot: (%d,%d)", bi->hotspot_x, bi->hotspot_y);
		}

		bi->bitsoffset = dbuf_getu32le(f, pos+10);
		de_dbg(c, "bits offset: %d", (int)bi->bitsoffset);
	}

	bmih_pos = pos + fhs;

	bi->infohdrsize = dbuf_getu32le(f, bmih_pos);

	if(bi->infohdrsize==0x474e5089 && (flags & DE_BMPINFO_ICO_FORMAT)) {
		// We don't examine PNG-formatted icons, but we can identify them.
		bi->infohdrsize = 0;
		bi->file_format = DE_BMPINFO_FMT_PNG;
		return 1;
	}

	de_dbg(c, "info header size: %d", (int)bi->infohdrsize);

	if(bi->infohdrsize==12) {
		bi->bytes_per_pal_entry = 3;
		bi->width = dbuf_getu16le(f, bmih_pos+4);
		bi->height = dbuf_getu16le(f, bmih_pos+6);
		bi->bitcount = dbuf_getu16le(f, bmih_pos+10);
	}
	else if(bi->infohdrsize>=16 && bi->infohdrsize<=124) {
		bi->bytes_per_pal_entry = 4;
		bi->width = dbuf_getu32le(f, bmih_pos+4);
		bi->height = dbuf_geti32le(f, bmih_pos+8);
		if(bi->height<0) {
			bi->is_topdown = 1;
			bi->height = -bi->height;
		}
		bi->bitcount = dbuf_getu16le(f, bmih_pos+14);
		if(bi->infohdrsize>=20) {
			bi->compression_field = (u32)dbuf_getu32le(f, bmih_pos+16);
			if(flags & DE_BMPINFO_CMPR_IS_4CC) {
				dbuf_read_fourcc(f, bmih_pos+16, &cmpr4cc, 4, 0x0);
			}
		}
		if(bi->infohdrsize>=24) {
			bi->sizeImage_field = dbuf_getu32le(f, bmih_pos+20);
		}
		if(bi->infohdrsize>=36) {
			bi->pal_entries = dbuf_getu32le(f, bmih_pos+32);
		}
	}
	else {
		return 0;
	}

	if(flags & DE_BMPINFO_ICO_FORMAT) bi->height /= 2;

	if(bi->bitcount>=1 && bi->bitcount<=8) {
		if(bi->pal_entries==0) {
			bi->pal_entries = de_pow2(bi->bitcount);
		}
		// I think the NumColors field (in icons) is supposed to be the maximum number of
		// colors implied by the bit depth, not the number of colors in the palette.
		bi->num_colors = de_pow2(bi->bitcount);
	}
	else {
		// An arbitrary value. All that matters is that it's >=256.
		bi->num_colors = 16777216;
	}

	de_dbg_dimensions(c, bi->width, bi->height);
	de_dbg(c, "bit count: %d", (int)bi->bitcount);

	if((flags & DE_BMPINFO_CMPR_IS_4CC) && (bi->compression_field>0xffff)) {
		de_snprintf(cmprname_dbgstr, sizeof(cmprname_dbgstr), "'%s'", cmpr4cc.id_dbgstr);
	}
	else {
		fmtutil_get_bmp_compression_name(bi->compression_field,
			cmprname_dbgstr, sizeof(cmprname_dbgstr), 0);
	}
	de_dbg(c, "compression: %u (%s)", (unsigned int)bi->compression_field, cmprname_dbgstr);

	if(bi->sizeImage_field!=0) {
		de_dbg(c, "sizeImage: %u", (unsigned int)bi->sizeImage_field);
	}

	de_dbg(c, "palette entries: %u", (unsigned int)bi->pal_entries);
	if(bi->pal_entries>256 && bi->bitcount>8) {
		de_warn(c, "Ignoring bad palette size (%u entries)", (unsigned int)bi->pal_entries);
		bi->pal_entries = 0;
	}

	bi->pal_bytes = bi->bytes_per_pal_entry*bi->pal_entries;
	bi->size_of_headers_and_pal = fhs + bi->infohdrsize + bi->pal_bytes;

	// FIXME: cmpr type 3 doesn't always mean BITFIELDS
	if(bi->compression_field==3) {
		bi->size_of_headers_and_pal += 12; // BITFIELDS
	}

	bi->is_compressed = !((bi->compression_field==0) ||
		(bi->compression_field==3 && bi->bitcount>1));

	if(!de_good_image_dimensions(c, bi->width, bi->height)) {
		return 0;
	}

	// TODO: This needs work, to decide how to handle compressed images.
	// TODO: What about BI_BITFIELDS images?
	if(bi->compression_field==0) {
		// Try to figure out the true size of the resource, minus any padding.

		bi->rowspan = ((bi->bitcount*bi->width +31)/32)*4;
		bi->foreground_size = bi->rowspan * bi->height;
		de_dbg(c, "foreground size: %d", (int)bi->foreground_size);

		if(flags & DE_BMPINFO_ICO_FORMAT) {
			bi->mask_rowspan = ((bi->width +31)/32)*4;
			bi->mask_size = bi->mask_rowspan * bi->height;
			de_dbg(c, "mask size: %d", (int)bi->mask_size);
		}
		else {
			bi->mask_size = 0;
		}

		bi->total_size = bi->size_of_headers_and_pal + bi->foreground_size + bi->mask_size;
	}
	else {
		// Don't try to figure out the true size of compressed or other unusual images.
		bi->total_size = len;
	}

	return 1;
}


void fmtutil_free_SAUCE(deark *c, struct de_SAUCE_info *si)
{
	if(!si) return;
	ucstring_destroy(si->title);
	ucstring_destroy(si->artist);
	ucstring_destroy(si->organization);
	ucstring_destroy(si->comment);
	de_free(c, si);
}


void fmtutil_macbitmap_read_rowbytes_and_bounds(deark *c, dbuf *f,
	struct fmtutil_macbitmap_info *bi, i64 pos)
{
	struct pict_rect tmprect;
	i64 rowbytes_code;

	de_dbg(c, "rowBytes/bounds part of bitmap/PixMap header, at %d", (int)pos);
	de_dbg_indent(c, 1);
	rowbytes_code = dbuf_getu16be(f, pos);
	bi->rowbytes = rowbytes_code & 0x7fff;
	bi->pixmap_flag = (rowbytes_code & 0x8000)?1:0;
	de_dbg(c, "rowBytes: %d", (int)bi->rowbytes);
	de_dbg(c, "pixmap flag: %d", bi->pixmap_flag);

	pict_read_rect(f, pos+2, &tmprect, "rect");
	bi->npwidth = tmprect.r - tmprect.l;
	bi->pdwidth = bi->npwidth; // default
	bi->height = tmprect.b - tmprect.t;

	de_dbg_indent(c, -1);
}


void fmtutil_macbitmap_read_baseaddr(deark *c, dbuf *f, struct fmtutil_macbitmap_info *bi, i64 pos)
{
	i64 n;
	de_dbg(c, "baseAddr part of PixMap, at %d", (int)pos);
	de_dbg_indent(c, 1);
	n = dbuf_getu32be(f, pos);
	de_dbg(c, "baseAddr: 0x%08x", (unsigned int)n);
	de_dbg_indent(c, -1);
}


void fmtutil_handle_SAUCE(deark *c, dbuf *f, struct de_SAUCE_info *si)
{
	de_module_params mparams;

	de_zeromem(&mparams, sizeof(de_module_params));
	mparams.out_params.obj1 = (void*)si;
	de_run_module_by_id_on_slice(c, "sauce", &mparams, f, 0, f->len);
}


static void pict_read_rect(dbuf *f, i64 pos,
	struct pict_rect *rect, const char *dbgname)
{
	rect->t = dbuf_geti16be(f, pos);
	rect->l = dbuf_geti16be(f, pos+2);
	rect->b = dbuf_geti16be(f, pos+4);
	rect->r = dbuf_geti16be(f, pos+6);

	if(dbgname) {
		de_dbg(f->c, "%s: (%d,%d)-(%d,%d)", dbgname, (int)rect->l, (int)rect->t,
			(int)rect->r, (int)rect->b);
	}
}


i64 fmtutil_hlp_get_cul_p(dbuf *f, i64 *ppos)
{
	i64 x1, x2;
	x1 = dbuf_getu16le_p(f, ppos);
	if(x1%2 == 0) {
		// If it's even, divide by two.
		return x1>>1;
	}
	// If it's odd, divide by two, and add 32768 times the value of
	// the next two bytes.
	x2 = dbuf_getu16le_p(f, ppos);
	return (x1>>1) | (x2<<15);
}


static int de_fmtutil_default_iff_chunk_handler(deark *c, struct de_iffctx *ictx)
{
	i64 dpos = ictx->chunkctx->dpos;
	i64 dlen = ictx->chunkctx->dlen;
	u32 chunktype = ictx->chunkctx->chunk4cc.id;

	switch(chunktype) {
		// Note that chunks appearing here should also be listed below,
		// in de_fmtutil_is_standard_iff_chunk().
	case CODE__c_:
		do_iff_text_chunk(c, ictx, dpos, dlen, "copyright");
		break;
	case CODE_ANNO:
		do_iff_anno(c, ictx, dpos, dlen);
		break;
	case CODE_AUTH:
		do_iff_text_chunk(c, ictx, dpos, dlen, "author");
		break;
	case CODE_NAME:
		do_iff_text_chunk(c, ictx, dpos, dlen, "name");
		break;
	case CODE_TEXT:
		do_iff_text_chunk(c, ictx, dpos, dlen, "text");
		break;
	}

	// Note we do not set ictx->handled. The caller is responsible for that.
	return 1;
}


struct de_SAUCE_info *fmtutil_create_SAUCE(deark *c)
{
	return de_malloc(c, sizeof(struct de_SAUCE_info));
}


static double pict_read_fixed(dbuf *f, i64 pos)
{
	i64 n;

	// I think QuickDraw's "Fixed point" numbers are signed, but I don't know
	// how negative numbers are handled.
	n = dbuf_geti32be(f, pos);
	return ((double)n)/65536.0;
}


void fmtutil_handle_plist(deark *c, dbuf *f, i64 pos, i64 len,
	de_finfo *fi, unsigned int flags)
{
	if(de_get_ext_option_bool(c, "extractplist", 0)) {
		dbuf_create_file_from_slice(f, pos, len,
			fi?NULL:"plist", fi, DE_CREATEFLAG_IS_AUX);
		return;
	}

	de_run_module_by_id_on_slice(c, "plist", NULL, f, pos, len);
}


void fmtutil_handle_exif2(deark *c, i64 pos, i64 len,
	u32 *returned_flags, u32 *orientation, u32 *exifversion)
{
	int user_opt;
	de_module_params *mparams = NULL;

	if(returned_flags) {
		*returned_flags = 0;
	}

	user_opt = de_get_ext_option_bool(c, "extractexif", -1);
	if(user_opt==1 || (c->extract_level>=2 && user_opt!=0)) {
		// Writing raw Exif data isn't very useful, but do so if requested.
		dbuf_create_file_from_slice(c->infile, pos, len, "exif.tif", NULL, DE_CREATEFLAG_IS_AUX);

		// Caller will have to reprocess the Exif file to extract anything from it.
		return;
	}

	mparams = de_malloc(c, sizeof(de_module_params));
	mparams->in_params.codes = "E";

	de_run_module_by_id_on_slice(c, "tiff", mparams, c->infile, pos, len);
	if(returned_flags) {
		// FIXME: It's an unfortunate bug that returned_flags does not work if
		// extract_level>=2, but for now there's no reasonable way to fix it.
		// We have to process -- not extract -- the Exif chunk if we want to
		// know what's in it.
		*returned_flags = mparams->out_params.flags;
		if((mparams->out_params.flags & 0x20) && orientation) {
			*orientation = mparams->out_params.uint1;
		}

		if((mparams->out_params.flags & 0x40) && exifversion) {
			*exifversion = mparams->out_params.uint2;
		}
	}

	de_free(c, mparams);
}


double dbuf_fmtutil_read_fixed_16_16(dbuf *f, i64 pos)
{
	i64 n;
	n = dbuf_geti32be(f, pos);
	return ((double)n)/65536.0;
}


static unsigned int spectrum512_FindIndex(i64 x, unsigned int c)
{
	i64 x1;

	x1 = 10 * (i64)c;

	if (c & 1)  /* If c is odd */
		x1 = x1 - 5;
	else        /* If c is even */
		x1 = x1 + 1;

	if (x >= x1 && x < x1+160)
		c = c + 16;
	else if (x >= x1+160)
		c = c + 32;

	return c;
}


void fmtutil_atari_set_standard_density(deark *c, struct atari_img_decode_data *adata,
	de_finfo *fi)
{
	switch(adata->bpp) {
	case 4:
		fi->density.code = DE_DENSITY_UNK_UNITS;
		fi->density.xdens = 240.0;
		fi->density.ydens = 200.0;
		break;
	case 2:
		fi->density.code = DE_DENSITY_UNK_UNITS;
		fi->density.xdens = 480.0;
		fi->density.ydens = 200.0;
		break;
	case 1:
		fi->density.code = DE_DENSITY_UNK_UNITS;
		fi->density.xdens = 480.0;
		fi->density.ydens = 400.0;
		break;
	}
}


i64 fmtutil_hlp_get_css_p(dbuf *f, i64 *ppos)
{
	i64 x1, x2;

	x1 = (i64)dbuf_getbyte_p(f, ppos);
	if(x1%2 == 0) {
		// If it's even, divide by two, and subtract 64
		return (x1>>1) - 64;
	}
	// If it's odd, divide by two, add 128 times the value of
	// the next byte, and subtract 16384.
	x1 >>= 1;
	x2 = (i64)dbuf_getbyte_p(f, ppos);
	x1 += x2 * 128;
	x1 -= 16384;
	return x1;
}


static void do_box_sequence(deark *c, struct de_boxesctx *bctx,
	i64 pos1, i64 len, i64 max_nboxes, int level)
{
	i64 pos;
	i64 box_len;
	i64 endpos;
	int ret;
	i64 box_count = 0;

	if(level >= 32) { // An arbitrary recursion limit.
		return;
	}

	pos = pos1;
	endpos = pos1 + len;

	while(pos < endpos) {
		if(max_nboxes>=0 && box_count>=max_nboxes) break;
		ret = do_box(c, bctx, pos, endpos-pos, level, &box_len);
		if(!ret) break;
		box_count++;
		pos += box_len;
	}
}


static int do_box(deark *c, struct de_boxesctx *bctx, i64 pos, i64 len,
	int level, i64 *pbytes_consumed)
{
	i64 size32, size64;
	i64 header_len; // Not including UUIDs
	i64 payload_len; // Including UUIDs
	i64 total_len;
	struct de_fourcc box4cc;
	char uuid_string[50];
	int ret;
	int retval = 0;
	struct de_boxdata *parentbox;
	struct de_boxdata *curbox;

	parentbox = bctx->curbox;
	bctx->curbox = de_malloc(c, sizeof(struct de_boxdata));
	curbox = bctx->curbox;
	curbox->parent = parentbox;

	if(len<8) {
		de_dbg(c, "(ignoring %d extra bytes at %"I64_FMT")", (int)len, pos);
		goto done;
	}

	size32 = dbuf_getu32be(bctx->f, pos);
	dbuf_read_fourcc(bctx->f, pos+4, &box4cc, 4, 0x0);
	curbox->boxtype = box4cc.id;

	if(size32>=8) {
		header_len = 8;
		payload_len = size32-8;
	}
	else if(size32==0) {
		header_len = 8;
		payload_len = len-8;
	}
	else if(size32==1) {
		if(len<16) {
			de_dbg(c, "(ignoring %d extra bytes at %"I64_FMT")", (int)len, pos);
			goto done;
		}
		header_len = 16;
		size64 = dbuf_geti64be(bctx->f, pos+8);
		if(size64<16) goto done;
		payload_len = size64-16;
	}
	else {
		de_err(c, "Invalid or unsupported box format");
		goto done;
	}

	total_len = header_len + payload_len;

	if(curbox->boxtype==DE_BOX_uuid && payload_len>=16) {
		curbox->is_uuid = 1;
		dbuf_read(bctx->f, curbox->uuid, pos+header_len, 16);
	}

	curbox->level = level;
	curbox->box_pos = pos;
	curbox->box_len = total_len;
	curbox->payload_pos = pos+header_len;
	curbox->payload_len = payload_len;
	if(curbox->is_uuid) {
		curbox->payload_pos += 16;
		curbox->payload_len -= 16;
	}

	if(bctx->identify_box_fn) {
		bctx->identify_box_fn(c, bctx);
	}

	if(c->debug_level>0) {
		char name_str[80];

		if(curbox->box_name) {
			de_snprintf(name_str, sizeof(name_str), " (%s)", curbox->box_name);
		}
		else {
			name_str[0] = '\0';
		}

		if(curbox->is_uuid) {
			fmtutil_render_uuid(c, curbox->uuid, uuid_string, sizeof(uuid_string));
			de_dbg(c, "box '%s'{%s}%s at %"I64_FMT", len=%"I64_FMT,
				box4cc.id_dbgstr, uuid_string, name_str,
				pos, total_len);
		}
		else {
			de_dbg(c, "box '%s'%s at %"I64_FMT", len=%"I64_FMT", dlen=%"I64_FMT,
				box4cc.id_dbgstr, name_str, pos,
				total_len, payload_len);
		}
	}

	if(total_len > len) {
		de_err(c, "Invalid oversized box, or unexpected end of file "
			"(box at %"I64_FMT" ends at %"I64_FMT", "
			"parent ends at %"I64_FMT")",
			pos, pos+total_len, pos+len);
		goto done;
	}

	de_dbg_indent(c, 1);
	ret = bctx->handle_box_fn(c, bctx);
	de_dbg_indent(c, -1);
	if(!ret) goto done;

	if(curbox->is_superbox) {
		i64 children_pos, children_len;
		i64 max_nchildren;

		de_dbg_indent(c, 1);
		children_pos = curbox->payload_pos + curbox->extra_bytes_before_children;
		children_len = curbox->payload_len - curbox->extra_bytes_before_children;
		max_nchildren = (curbox->num_children_is_known) ? curbox->num_children : -1;
		do_box_sequence(c, bctx, children_pos, children_len, max_nchildren, level+1);
		de_dbg_indent(c, -1);
	}

	*pbytes_consumed = total_len;
	retval = 1;

done:
	de_free(c, bctx->curbox);
	bctx->curbox = parentbox; // Restore the curbox pointer
	return retval;
}


static int do_iff_chunk_sequence(deark *c, struct de_iffctx *ictx,
	i64 pos1, i64 len, int level)
{
	i64 pos;
	i64 endpos;
	i64 chunk_len;
	struct de_fourcc saved_container_fmt4cc;
	struct de_fourcc saved_container_contentstype4cc;
	int ret;

	if(level >= 16) { // An arbitrary recursion limit.
		return 0;
	}

	endpos = pos1+len;
	saved_container_fmt4cc = ictx->curr_container_fmt4cc;
	saved_container_contentstype4cc = ictx->curr_container_contentstype4cc;

	pos = pos1;
	while(pos < endpos) {
		ictx->curr_container_fmt4cc = saved_container_fmt4cc;
		ictx->curr_container_contentstype4cc = saved_container_contentstype4cc;

		if(ictx->handle_nonchunk_data_fn) {
			i64 skip_len = 0;
			ret = ictx->handle_nonchunk_data_fn(c, ictx, pos, &skip_len);
			if(ret && skip_len>0) {
				pos += de_pad_to_n(skip_len, ictx->alignment);
				continue;
			}
		}

		ret = do_iff_chunk(c, ictx, pos, endpos-pos, level, &chunk_len);
		if(!ret) return 0;
		pos += chunk_len;
	}

	ictx->curr_container_fmt4cc = saved_container_fmt4cc;
	ictx->curr_container_contentstype4cc = saved_container_contentstype4cc;

	return 1;
}


void fmtutil_get_bmp_compression_name(u32 code, char *s, size_t s_len,
	int is_os2v2)
{
	const char *name1 = "?";
	switch(code) {
	case 0: name1 = "BI_RGB, uncompressed"; break;
	case 1: name1 = "BI_RLE8"; break;
	case 2: name1 = "BI_RLE4"; break;
	case 3:
		if(is_os2v2)
			name1 = "Huffman 1D";
		else
			name1 = "BI_BITFIELDS, uncompressed";
		break;
	case 4:
		if(is_os2v2)
			name1 = "RLE24";
		else
			name1 = "BI_JPEG";
		break;
	case 5: name1 = "BI_PNG"; break;
	}
	de_strlcpy(s, name1, s_len);
}


const char *fmtutil_tiff_orientation_name(i64 n)
{
	static const char *names[9] = {
		"?", "top-left", "top-right", "bottom-right", "bottom-left",
		"left-top", "right-top", "right-bottom", "left-bottom"
	};
	if(n>=1 && n<=8) return names[n];
	return names[0];
}


void fmtutil_macbitmap_read_pixmap_only_fields(deark *c, dbuf *f, struct fmtutil_macbitmap_info *bi,
	i64 pos)
{
	i64 pixmap_version;
	i64 pack_size;
	i64 plane_bytes;
	i64 n;

	de_dbg(c, "additional PixMap header fields, at %d", (int)pos);
	de_dbg_indent(c, 1);

	pixmap_version = dbuf_getu16be(f, pos+0);
	de_dbg(c, "pixmap version: %d", (int)pixmap_version);

	bi->packing_type = dbuf_getu16be(f, pos+2);
	de_dbg(c, "packing type: %d", (int)bi->packing_type);

	pack_size = dbuf_getu32be(f, pos+4);
	de_dbg(c, "pixel data length: %d", (int)pack_size);

	bi->hdpi = pict_read_fixed(f, pos+8);
	bi->vdpi = pict_read_fixed(f, pos+12);
	de_dbg(c, "dpi: %.2f"DE_CHAR_TIMES"%.2f", bi->hdpi, bi->vdpi);

	bi->pixeltype = dbuf_getu16be(f, pos+16);
	bi->pixelsize = dbuf_getu16be(f, pos+18);
	bi->cmpcount = dbuf_getu16be(f, pos+20);
	bi->cmpsize = dbuf_getu16be(f, pos+22);
	de_dbg(c, "pixel type=%d, bits/pixel=%d, components/pixel=%d, bits/comp=%d",
		(int)bi->pixeltype, (int)bi->pixelsize, (int)bi->cmpcount, (int)bi->cmpsize);

	if(bi->pixelsize>0) {
		bi->pdwidth = (bi->rowbytes*8)/bi->pixelsize;
	}
	if(bi->pdwidth < bi->npwidth) {
		bi->pdwidth = bi->npwidth;
	}

	plane_bytes = dbuf_getu32be(f, pos+24);
	de_dbg(c, "plane bytes: %d", (int)plane_bytes);

	bi->pmTable = (u32)dbuf_getu32be(f, pos+28);
	de_dbg(c, "pmTable: 0x%08x", (unsigned int)bi->pmTable);

	n = dbuf_getu32be(f, pos+32);
	de_dbg(c, "pmReserved: 0x%08x", (unsigned int)n);

	de_dbg_indent(c, -1);
}


int fmtutil_detect_SAUCE(deark *c, dbuf *f, struct de_SAUCE_detection_data *sdd,
	unsigned int flags)
{
	de_zeromem(sdd, sizeof(struct de_SAUCE_detection_data));
	if(f->len<128) return 0;
	if(dbuf_memcmp(f, f->len-128, "SAUCE00", 7)) return 0;
	if(flags & 0x1) {
		de_dbg(c, "SAUCE metadata, signature at %"I64_FMT, f->len-128);
	}
	sdd->has_SAUCE = 1;
	sdd->data_type = dbuf_getbyte(f, f->len-128+94);
	sdd->file_type = dbuf_getbyte(f, f->len-128+95);
	return (int)sdd->has_SAUCE;
}


void fmtutil_riscos_read_load_exec(deark *c, dbuf *f, struct de_riscos_file_attrs *rfa, i64 pos1)
{
	i64 pos = pos1;

	rfa->load_addr = (u32)dbuf_getu32le_p(f, &pos);
	rfa->exec_addr = (u32)dbuf_getu32le_p(f, &pos);
	de_dbg(c, "load/exec addrs: 0x%08x, 0x%08x", (unsigned int)rfa->load_addr,
		(unsigned int)rfa->exec_addr);
	de_dbg_indent(c, 1);
	if((rfa->load_addr&0xfff00000U)==0xfff00000U) {
		rfa->file_type = (unsigned int)((rfa->load_addr&0xfff00)>>8);
		rfa->file_type_known = 1;
		de_dbg(c, "file type: %03X", rfa->file_type);

		de_riscos_loadexec_to_timestamp(rfa->load_addr, rfa->exec_addr, &rfa->mod_time);
		dbg_timestamp(c, &rfa->mod_time, "timestamp");
	}
	de_dbg_indent(c, -1);
}


static void wrap_in_tiff(deark *c, dbuf *f, i64 dpos, i64 dlen,
	const char *swstring, unsigned int tag, const char *ext, unsigned int createflags)
{
	dbuf *outf = NULL;
	i64 ifdoffs;
	i64 sw_len, sw_len_padded;
	i64 data_len_padded;

	sw_len = 1+(i64)de_strlen(swstring);
	if(sw_len<=4) return;
	sw_len_padded = de_pad_to_2(sw_len);

	if(dlen>4) {
		data_len_padded = de_pad_to_2(dlen);
	}
	else {
		data_len_padded = 0;
	}

	outf = dbuf_create_output_file(c, ext, NULL, 0);
	dbuf_write(outf, (const u8*)"\x4d\x4d\x00\x2a", 4);
	ifdoffs = 8 + sw_len_padded + data_len_padded;
	dbuf_writeu32be(outf, ifdoffs);
	dbuf_write(outf, (const u8*)swstring, sw_len);
	if(sw_len%2) dbuf_writebyte(outf, 0);
	if(dlen>4) {
		dbuf_copy(f, dpos, dlen, outf);
		if(dlen%2) dbuf_writebyte(outf, 0);
	}

	dbuf_writeu16be(outf, 2); // number of dir entries;

	dbuf_writeu16be(outf, 305); // Software tag
	dbuf_writeu16be(outf, 2); // type=ASCII
	dbuf_writeu32be(outf, sw_len);
	dbuf_writeu32be(outf, 8); // offset

	dbuf_writeu16be(outf, (i64)tag);
	dbuf_writeu16be(outf, 1);
	dbuf_writeu32be(outf, dlen);
	if(dlen>4) {
		dbuf_writeu32be(outf, 8+sw_len_padded);
	}
	else {
		dbuf_copy(f, dpos, dlen, outf);
		dbuf_write_zeroes(outf, 4-dlen);
	}

	dbuf_writeu32be(outf, 0); // end of IFD
	dbuf_close(outf);
}


static void fourcc_clear(struct de_fourcc *fourcc)
{
	de_zeromem(fourcc, sizeof(struct de_fourcc));
}


 */
static void php_wddx_pop_element(void *user_data, const XML_Char *name)
{
	st_entry 			*ent1, *ent2;
	wddx_stack 			*stack = (wddx_stack *)user_data;
	HashTable 			*target_hash;
	zend_class_entry 	*pce;
	zval				obj;

/* OBJECTS_FIXME */
	if (stack->top == 0) {
		return;
	}

	if (!strcmp((char *)name, EL_STRING) || !strcmp((char *)name, EL_NUMBER) ||
		!strcmp((char *)name, EL_BOOLEAN) || !strcmp((char *)name, EL_NULL) ||
	  	!strcmp((char *)name, EL_ARRAY) || !strcmp((char *)name, EL_STRUCT) ||
		!strcmp((char *)name, EL_RECORDSET) || !strcmp((char *)name, EL_BINARY) ||
		!strcmp((char *)name, EL_DATETIME)) {
		wddx_stack_top(stack, (void**)&ent1);

		if (Z_TYPE(ent1->data) == IS_UNDEF) {
			if (stack->top > 1) {
				stack->top--;
				efree(ent1);
			} else {
				stack->done = 1;
			}
			return;
		}

		if (!strcmp((char *)name, EL_BINARY)) {
			zend_string *new_str = php_base64_decode(
				(unsigned char *)Z_STRVAL(ent1->data), Z_STRLEN(ent1->data));
			zval_ptr_dtor(&ent1->data);
			ZVAL_STR(&ent1->data, new_str);
		}

		/* Call __wakeup() method on the object. */
		if (Z_TYPE(ent1->data) == IS_OBJECT) {
			zval fname, retval;

			ZVAL_STRING(&fname, "__wakeup");

			call_user_function_ex(NULL, &ent1->data, &fname, &retval, 0, 0, 0, NULL);

			zval_ptr_dtor(&fname);
			zval_ptr_dtor(&retval);
		}

		if (stack->top > 1) {
			stack->top--;
			wddx_stack_top(stack, (void**)&ent2);

			/* if non-existent field */
			if (ent2->type == ST_FIELD && Z_ISUNDEF(ent2->data)) {
				zval_ptr_dtor(&ent1->data);
				efree(ent1);
				return;
			}

			if (Z_TYPE(ent2->data) == IS_ARRAY || Z_TYPE(ent2->data) == IS_OBJECT) {
				target_hash = HASH_OF(&ent2->data);

				if (ent1->varname) {
					if (!strcmp(ent1->varname, PHP_CLASS_NAME_VAR) &&
						Z_TYPE(ent1->data) == IS_STRING && Z_STRLEN(ent1->data) &&
						ent2->type == ST_STRUCT && Z_TYPE(ent2->data) == IS_ARRAY) {
						zend_bool incomplete_class = 0;

						zend_str_tolower(Z_STRVAL(ent1->data), Z_STRLEN(ent1->data));
						zend_string_forget_hash_val(Z_STR(ent1->data));
						if ((pce = zend_hash_find_ptr(EG(class_table), Z_STR(ent1->data))) == NULL) {
							incomplete_class = 1;
							pce = PHP_IC_ENTRY;
						}

						/* Initialize target object */
						object_init_ex(&obj, pce);

						/* Merge current hashtable with object's default properties */
						zend_hash_merge(Z_OBJPROP(obj),
										Z_ARRVAL(ent2->data),
										zval_add_ref, 0);

						if (incomplete_class) {
							php_store_class_name(&obj, Z_STRVAL(ent1->data), Z_STRLEN(ent1->data));
						}

						/* Clean up old array entry */
						zval_ptr_dtor(&ent2->data);

						/* Set stack entry to point to the newly created object */
						ZVAL_COPY_VALUE(&ent2->data, &obj);

						/* Clean up class name var entry */
						zval_ptr_dtor(&ent1->data);
					} else if (Z_TYPE(ent2->data) == IS_OBJECT) {
						zend_class_entry *old_scope = EG(scope);

						EG(scope) = Z_OBJCE(ent2->data);
						add_property_zval(&ent2->data, ent1->varname, &ent1->data);
						if Z_REFCOUNTED(ent1->data) Z_DELREF(ent1->data);
						EG(scope) = old_scope;
					} else {
						zend_symtable_str_update(target_hash, ent1->varname, strlen(ent1->varname), &ent1->data);
					}
					efree(ent1->varname);
				} else	{
					zend_hash_next_index_insert(target_hash, &ent1->data);
				}
			}
			efree(ent1);
		} else {
			stack->done = 1;
		}
	} else if (!strcmp((char *)name, EL_VAR) && stack->varname) {
		efree(stack->varname);
		stack->varname = NULL;
	} else if (!strcmp((char *)name, EL_FIELD)) {
		st_entry *ent;
		wddx_stack_top(stack, (void **)&ent);
		efree(ent);
		stack->top--;
	}

Fix for bug #72790 and bug #72799

(cherry picked from commit a14fdb9746262549bbbb96abb87338bacd147e1b)

Conflicts:
	ext/wddx/wddx.c


 */
void php_wddx_serialize_var(wddx_packet *packet, zval *var, zend_string *name)
{
	HashTable *ht;

	if (name) {
		char *tmp_buf;
		zend_string *name_esc = php_escape_html_entities((unsigned char *) ZSTR_VAL(name), ZSTR_LEN(name), 0, ENT_QUOTES, NULL);
		tmp_buf = emalloc(ZSTR_LEN(name_esc) + sizeof(WDDX_VAR_S));
		snprintf(tmp_buf, ZSTR_LEN(name_esc) + sizeof(WDDX_VAR_S), WDDX_VAR_S, ZSTR_VAL(name_esc));
		php_wddx_add_chunk(packet, tmp_buf);
		efree(tmp_buf);
		zend_string_release(name_esc);
	}

	if (Z_TYPE_P(var) == IS_INDIRECT) {
		var = Z_INDIRECT_P(var);
	}
	ZVAL_DEREF(var);
	switch (Z_TYPE_P(var)) {
		case IS_STRING:
			php_wddx_serialize_string(packet, var);
			break;

		case IS_LONG:
		case IS_DOUBLE:
			php_wddx_serialize_number(packet, var);
			break;

		case IS_TRUE:
		case IS_FALSE:
			php_wddx_serialize_boolean(packet, var);
			break;

		case IS_NULL:
			php_wddx_serialize_unset(packet);
			break;

		case IS_ARRAY:
			ht = Z_ARRVAL_P(var);
			if (ht->u.v.nApplyCount > 1) {
				php_error_docref(NULL, E_RECOVERABLE_ERROR, "WDDX doesn't support circular references");
				return;
			}
			if (ZEND_HASH_APPLY_PROTECTION(ht)) {
				ht->u.v.nApplyCount++;
			}
			php_wddx_serialize_array(packet, var);
			if (ZEND_HASH_APPLY_PROTECTION(ht)) {
				ht->u.v.nApplyCount--;
			}
			break;

		case IS_OBJECT:
			ht = Z_OBJPROP_P(var);
			if (ht->u.v.nApplyCount > 1) {
				php_error_docref(NULL, E_RECOVERABLE_ERROR, "WDDX doesn't support circular references");
				return;
			}
			ht->u.v.nApplyCount++;
 			php_wddx_serialize_object(packet, var);
			ht->u.v.nApplyCount--;
			break;
	}

	if (name) {
		php_wddx_add_chunk_static(packet, WDDX_VAR_E);
	}


   Creates a new packet and serializes given variables into a struct */
PHP_FUNCTION(wddx_serialize_vars)
{
	int num_args, i;
	wddx_packet *packet;
	zval *args = NULL;

	if (zend_parse_parameters(ZEND_NUM_ARGS(), "+", &args, &num_args) == FAILURE) {
		return;
	}

	packet = php_wddx_constructor();

	php_wddx_packet_start(packet, NULL, 0);
	php_wddx_add_chunk_static(packet, WDDX_STRUCT_S);

	for (i=0; i<num_args; i++) {
		zval *arg;
		if (!Z_ISREF(args[i])) {
			arg = &args[i];
		} else {
			arg = Z_REFVAL(args[i]);
		}
		if (Z_TYPE_P(arg) != IS_ARRAY && Z_TYPE_P(arg) != IS_OBJECT) {
			convert_to_string_ex(arg);
		}
		php_wddx_add_var(packet, arg);
	}

	php_wddx_add_chunk_static(packet, WDDX_STRUCT_E);
	php_wddx_packet_end(packet);
	smart_str_0(packet);

	RETVAL_STR_COPY(packet->s);
	php_wddx_destructor(packet);


 */
static void php_wddx_serialize_unset(wddx_packet *packet)
{
	php_wddx_add_chunk_static(packet, WDDX_NULL);


 */
PHP_MINFO_FUNCTION(wddx)
{
	php_info_print_table_start();
#if HAVE_PHP_SESSION && !defined(COMPILE_DL_SESSION)
	php_info_print_table_header(2, "WDDX Support", "enabled" );
	php_info_print_table_row(2, "WDDX Session Serializer", "enabled" );
#else
	php_info_print_table_row(2, "WDDX Support", "enabled" );
#endif
	php_info_print_table_end();


 */
static void php_wddx_add_var(wddx_packet *packet, zval *name_var)
{
	zval *val;
	HashTable *target_hash;

	if (Z_TYPE_P(name_var) == IS_STRING) {
		zend_array *symbol_table = zend_rebuild_symbol_table();
		if ((val = zend_hash_find(symbol_table, Z_STR_P(name_var))) != NULL) {
			if (Z_TYPE_P(val) == IS_INDIRECT) {
				val = Z_INDIRECT_P(val);
			}
			php_wddx_serialize_var(packet, val, Z_STR_P(name_var));
		}
	} else if (Z_TYPE_P(name_var) == IS_ARRAY || Z_TYPE_P(name_var) == IS_OBJECT)	{
		int is_array = Z_TYPE_P(name_var) == IS_ARRAY;

		target_hash = HASH_OF(name_var);

		if (is_array && target_hash->u.v.nApplyCount > 1) {
			php_error_docref(NULL, E_WARNING, "recursion detected");
			return;
		}

		if (Z_IMMUTABLE_P(name_var)) {
			ZEND_HASH_FOREACH_VAL(target_hash, val) {
				php_wddx_add_var(packet, val);
			} ZEND_HASH_FOREACH_END();
		} else {
			ZEND_HASH_FOREACH_VAL(target_hash, val) {
				if (is_array) {
					target_hash->u.v.nApplyCount++;
				}

				ZVAL_DEREF(val);
				php_wddx_add_var(packet, val);

				if (is_array) {
					target_hash->u.v.nApplyCount--;
				}
			} ZEND_HASH_FOREACH_END();
		}
	}


 */
static int wddx_stack_push(wddx_stack *stack, void *element, int size)
{
	if (stack->top >= stack->max) {		/* we need to allocate more memory */
		stack->elements = (void **) erealloc(stack->elements,
				   (sizeof(void **) * (stack->max += STACK_BLOCK_SIZE)));
	}
	stack->elements[stack->top] = (void *) emalloc(size);
	memcpy(stack->elements[stack->top], element, size);
	return stack->top++;


 */
static void php_wddx_serialize_number(wddx_packet *packet, zval *var)
{
	char tmp_buf[WDDX_BUF_LEN];
	zend_string *str = zval_get_string(var);
	snprintf(tmp_buf, sizeof(tmp_buf), WDDX_NUMBER, ZSTR_VAL(str));
	zend_string_release(str);

	php_wddx_add_chunk(packet, tmp_buf);


   Ends specified WDDX packet and returns the string containing the packet */
PHP_FUNCTION(wddx_packet_end)
{
	zval *packet_id;
	wddx_packet *packet = NULL;

	if (zend_parse_parameters(ZEND_NUM_ARGS(), "r", &packet_id) == FAILURE) {
		return;
	}

	if ((packet = (wddx_packet *)zend_fetch_resource(Z_RES_P(packet_id), "WDDX packet ID", le_wddx)) == NULL) {
		RETURN_FALSE;
	}

	php_wddx_add_chunk_static(packet, WDDX_STRUCT_E);

	php_wddx_packet_end(packet);
	smart_str_0(packet);

	RETVAL_STR_COPY(packet->s);

	zend_list_close(Z_RES_P(packet_id));


 */
static int wddx_stack_is_empty(wddx_stack *stack)
{
	if (stack->top == 0) {
		return 1;
	} else {
		return 0;
	}


 */
void php_wddx_packet_end(wddx_packet *packet)
{
	php_wddx_add_chunk_static(packet, WDDX_DATA_E);
	php_wddx_add_chunk_static(packet, WDDX_PACKET_E);


 */
static void release_wddx_packet_rsrc(zend_resource *rsrc)
{
	smart_str *str = (smart_str *)rsrc->ptr;
	smart_str_free(str);
	efree(str);


 */
void php_wddx_destructor(wddx_packet *packet)
{
	smart_str_free(packet);
	efree(packet);


 */
static void php_wddx_serialize_array(wddx_packet *packet, zval *arr)
{
	zval *ent;
	zend_string *key;
	int is_struct = 0;
	zend_ulong idx;
	HashTable *target_hash;
	char tmp_buf[WDDX_BUF_LEN];
	zend_ulong ind = 0;

	target_hash = Z_ARRVAL_P(arr);
	ZEND_HASH_FOREACH_KEY(target_hash, idx, key) {
		if (key) {
			is_struct = 1;
			break;
		}

		if (idx != ind) {
			is_struct = 1;
			break;
		}
		ind++;
	} ZEND_HASH_FOREACH_END();

	if (is_struct) {
		php_wddx_add_chunk_static(packet, WDDX_STRUCT_S);
	} else {
		snprintf(tmp_buf, sizeof(tmp_buf), WDDX_ARRAY_S, zend_hash_num_elements(target_hash));
		php_wddx_add_chunk(packet, tmp_buf);
	}

	ZEND_HASH_FOREACH_KEY_VAL(target_hash, idx, key, ent) {
		if (ent == arr) {
			continue;
		}

		if (is_struct) {
			if (key) {
				php_wddx_serialize_var(packet, ent, key);
			} else {
				key = zend_long_to_str(idx);
				php_wddx_serialize_var(packet, ent, key);
				zend_string_release(key);
			}
		} else {
			php_wddx_serialize_var(packet, ent, NULL);
		}
	} ZEND_HASH_FOREACH_END();

	if (is_struct) {
		php_wddx_add_chunk_static(packet, WDDX_STRUCT_E);
	} else {
		php_wddx_add_chunk_static(packet, WDDX_ARRAY_E);
	}


 */
static void php_wddx_serialize_object(wddx_packet *packet, zval *obj)
{
/* OBJECTS_FIXME */
	zval *ent, fname, *varname;
	zval retval;
	zend_string *key;
	zend_ulong idx;
	char tmp_buf[WDDX_BUF_LEN];
	HashTable *objhash, *sleephash;

	ZVAL_STRING(&fname, "__sleep");
	/*
	 * We try to call __sleep() method on object. It's supposed to return an
	 * array of property names to be serialized.
	 */
	if (call_user_function_ex(CG(function_table), obj, &fname, &retval, 0, 0, 1, NULL) == SUCCESS) {
		if (!Z_ISUNDEF(retval) && (sleephash = HASH_OF(&retval))) {
			PHP_CLASS_ATTRIBUTES;

			PHP_SET_CLASS_ATTRIBUTES(obj);

			php_wddx_add_chunk_static(packet, WDDX_STRUCT_S);
			snprintf(tmp_buf, WDDX_BUF_LEN, WDDX_VAR_S, PHP_CLASS_NAME_VAR);
			php_wddx_add_chunk(packet, tmp_buf);
			php_wddx_add_chunk_static(packet, WDDX_STRING_S);
			php_wddx_add_chunk_ex(packet, ZSTR_VAL(class_name), ZSTR_LEN(class_name));
			php_wddx_add_chunk_static(packet, WDDX_STRING_E);
			php_wddx_add_chunk_static(packet, WDDX_VAR_E);

			PHP_CLEANUP_CLASS_ATTRIBUTES();

			objhash = Z_OBJPROP_P(obj);

			ZEND_HASH_FOREACH_VAL(sleephash, varname) {
				if (Z_TYPE_P(varname) != IS_STRING) {
					php_error_docref(NULL, E_NOTICE, "__sleep should return an array only containing the names of instance-variables to serialize.");
					continue;
				}

				if ((ent = zend_hash_find(objhash, Z_STR_P(varname))) != NULL) {
					php_wddx_serialize_var(packet, ent, Z_STR_P(varname));
				}
			} ZEND_HASH_FOREACH_END();

			php_wddx_add_chunk_static(packet, WDDX_STRUCT_E);
		}
	} else {
		PHP_CLASS_ATTRIBUTES;

		PHP_SET_CLASS_ATTRIBUTES(obj);

		php_wddx_add_chunk_static(packet, WDDX_STRUCT_S);
		snprintf(tmp_buf, WDDX_BUF_LEN, WDDX_VAR_S, PHP_CLASS_NAME_VAR);
		php_wddx_add_chunk(packet, tmp_buf);
		php_wddx_add_chunk_static(packet, WDDX_STRING_S);
		php_wddx_add_chunk_ex(packet, ZSTR_VAL(class_name), ZSTR_LEN(class_name));
		php_wddx_add_chunk_static(packet, WDDX_STRING_E);
		php_wddx_add_chunk_static(packet, WDDX_VAR_E);

		PHP_CLEANUP_CLASS_ATTRIBUTES();

		objhash = Z_OBJPROP_P(obj);
		ZEND_HASH_FOREACH_KEY_VAL(objhash, idx, key, ent) {
			if (ent == obj) {
				continue;
			}
			if (key) {
				const char *class_name, *prop_name;
				size_t prop_name_len;
				zend_string *tmp;

				zend_unmangle_property_name_ex(key, &class_name, &prop_name, &prop_name_len);
				tmp = zend_string_init(prop_name, prop_name_len, 0);
				php_wddx_serialize_var(packet, ent, tmp);
				zend_string_release(tmp);
			} else {
				key = zend_long_to_str(idx);
				php_wddx_serialize_var(packet, ent, key);
				zend_string_release(key);
			}
		} ZEND_HASH_FOREACH_END();
		php_wddx_add_chunk_static(packet, WDDX_STRUCT_E);
	}

	zval_ptr_dtor(&fname);
	zval_ptr_dtor(&retval);