diff --git a/Gemfile b/Gemfile index d83e9429d..3f9023901 100644 --- a/Gemfile +++ b/Gemfile @@ -73,7 +73,7 @@ gem 'ruby-progressbar' # own gems gem 'quintel_merit', ref: 'aae77e0', github: 'quintel/merit' -gem 'atlas', ref: '72dbea1', github: 'quintel/atlas' #TODO: Update to latest merge commit once Atlas branch is merged into master +gem 'atlas', ref: 'b2d47a1', github: 'quintel/atlas' gem 'fever', ref: '2afebd1', github: 'quintel/fever' gem 'refinery', ref: '36b8e34', github: 'quintel/refinery' gem 'rubel', ref: '9fe7010', github: 'quintel/rubel' diff --git a/Gemfile.lock b/Gemfile.lock index 3ffc5dd64..1d8757f4d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ GIT remote: https://github.com/quintel/atlas.git - revision: 72dbea148d09aaae4b38361203f6e98bb29a9efb - ref: 72dbea1 + revision: b2d47a1495f85638398d89acc17c6a2da7d2acac + ref: b2d47a1 specs: atlas (1.0.0) activemodel (>= 7) diff --git a/app/controllers/inspect/gqueries_controller.rb b/app/controllers/inspect/gqueries_controller.rb index 6dffce086..85981933f 100644 --- a/app/controllers/inspect/gqueries_controller.rb +++ b/app/controllers/inspect/gqueries_controller.rb @@ -18,7 +18,7 @@ def test redirect_to inspect_debug_gql_path(gquery: params[:query]) elsif params[:query].present? begin - @result = @gql.query(params[:query].gsub(/\s/,''), nil, true) + @result = @gql.query(params[:query], nil, true) rescue Gql::CommandError => ex @error = ex end diff --git a/app/models/etsource/sectors.rb b/app/models/etsource/sectors.rb new file mode 100644 index 000000000..72b590914 --- /dev/null +++ b/app/models/etsource/sectors.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +module Etsource + # Reads the sector mapping and node labels from Atlas and caches the + # structures GQL and the configured CSV exports need. + # + # * mapping - {scheme => {value => [[label, use], ...]}}, for + # `SECTOR(scheme, value)` style GQL queries. + # * node_index(g) - {[label, use] => [node_key, ...]} for one graph type, + # with `use` baked in at import so it is never consulted + # per query. + # * raw_rows - mapping rows in file order with raw display values, + # for mapping-driven CSV exports. + class Sectors + # Public: The inverted mapping index, keyed by scheme then normalized value. + def mapping + NastyCache.instance.fetch('sector_mapping_hash') do + Atlas::SectorMapping.load.to_h + end + end + + # Public: The (label, use) -> node keys index for the given graph type + # (:energy or :molecules). + def node_index(graph_type) + NastyCache.instance.fetch("sector_node_index_#{graph_type}") do + build_node_index(node_class_for(graph_type)) + end + end + + # Public: Every mapping row in file order, as an Atlas::SectorMapping::RawRow + # (the (label, use) pair plus the raw display value of every scheme cell). + def raw_rows + NastyCache.instance.fetch('sector_raw_rows') do + Atlas::SectorMapping.load.raw_rows + end + end + + private + + def build_node_index(node_class) + node_class.all.each_with_object({}) do |node, index| + next if node.sector_label.nil? + + pair = [node.sector_label, Atlas::SectorMapping.normalize(node.use)] + (index[pair] ||= []) << node.key + end + end + + def node_class_for(graph_type) + graph_type.to_sym == :molecules ? Atlas::MoleculeNode : Atlas::EnergyNode + end + end +end diff --git a/app/models/gql/command.rb b/app/models/gql/command.rb index 653333df1..73546f37d 100644 --- a/app/models/gql/command.rb +++ b/app/models/gql/command.rb @@ -108,12 +108,15 @@ def truncate(string, length = 40) string.length > (length - 3) ? "#{ string[0..length] }..." : string end - # Internal: Removes extra "artifacts" from the source. + # Internal: Removes extra "artifacts" from the source. Whitespace is + # insignificant in GQL and stripped, except inside string literals, where + # it may be meaningful (e.g. SECTOR scheme values such as 'Fuels + # production'). def clean(string) cleaned = (string || '').dup - cleaned.gsub!(/[\n\s\t]/, '') - cleaned.gsub!(/^[a-z]+\:/,'') + cleaned.gsub!(/('[^']*'|"[^"]*")|\s+/) { Regexp.last_match(1) || '' } + cleaned.sub!(/\A[a-z]+\:/, '') cleaned end diff --git a/app/models/gql/gql_error.rb b/app/models/gql/gql_error.rb index 462b1a5de..d016999cf 100644 --- a/app/models/gql/gql_error.rb +++ b/app/models/gql/gql_error.rb @@ -23,4 +23,21 @@ def initialize(curve, attribute = nil) end end + # Raised when a SECTOR/MSECTOR/EMISSIONS scheme-form query names a + # classification scheme which does not exist in the sector mapping. + class UnknownSectorSchemeError < GqlError + def initialize(scheme, valid_schemes) + super("Unknown sector mapping scheme #{scheme.inspect}. " \ + "Valid schemes: #{valid_schemes.map(&:inspect).join(', ')}.") + end + end + + # Raised when a SECTOR/MSECTOR/EMISSIONS scheme-form query names a value + # which does not appear in that scheme's column. + class UnknownSectorValueError < GqlError + def initialize(scheme, value) + super("Unknown value #{value.inspect} for sector mapping scheme " \ + "#{scheme.inspect}.") + end + end end diff --git a/app/models/gql/query_interface/lookup.rb b/app/models/gql/query_interface/lookup.rb index 48f4aa88d..2985cf9ef 100644 --- a/app/models/gql/query_interface/lookup.rb +++ b/app/models/gql/query_interface/lookup.rb @@ -64,10 +64,30 @@ def molecule_sector_nodes(keys) molecule_graph_helper.sector_nodes(keys) end + # Scheme-based sector mapping lookups. `energy`/`molecule` select the + # graph whose live nodes are returned; the mapping itself is shared. + def energy_sector_node_map(scheme, values) + graph.sector_map.lookup(scheme, values) + end + + def molecule_sector_map(scheme, values) + molecules.sector_map.lookup(scheme, values) + end + + # The energy graph's resolver, used for EMISSIONS scheme dispatch and + # pair resolution (the mapping is graph-independent). + def sector_resolver + graph.sector_map + end + def energy_use_nodes(keys) energy_graph_helper.use_nodes(keys) end + def molecule_use_nodes(keys) + molecule_graph_helper.use_nodes(keys) + end + def group_energy_nodes(keys) energy_graph_helper.group_nodes(keys) end diff --git a/app/models/gql/runtime/functions/lookup.rb b/app/models/gql/runtime/functions/lookup.rb index 6c979bdad..d878350df 100644 --- a/app/models/gql/runtime/functions/lookup.rb +++ b/app/models/gql/runtime/functions/lookup.rb @@ -91,13 +91,13 @@ def MALL def GROUP(*keys) scope.group_energy_nodes(keys) end - alias G GROUP + alias_method :G, :GROUP # Returns an Array of {Qernel::Node} for given molecule group. See GROUP. def MGROUP(*keys) scope.group_molecule_nodes(keys) end - alias MG MGROUP + alias_method :MG, :MGROUP # Returns an Array of {Qernel::Edges} for given energy group. # @@ -108,33 +108,48 @@ def MGROUP(*keys) def EDGE_GROUP(*keys) scope.group_energy_edges(keys) end - alias EG EDGE_GROUP + alias_method :EG, :EDGE_GROUP # Returns an Array of {Qernel::Edges} for given molecule group. See EDGE_GROUP. def MEDGE_GROUP(*keys) scope.group_molecule_edges(keys) end - alias MEG EDGE_GROUP + alias_method :MEG, :MEDGE_GROUP - # Returns an Array of {Qernel::Node} for given energy sector. + # Returns an Array of {Qernel::Node} for an energy sector. # - # Examples + # Dispatches on arity: # - # SECTOR(households) + # * One argument - namespace-sector filter. + # SECTOR(households) + # * Two or more - a classification scheme followed by one or more + # values; returns the union of matching nodes. + # SECTOR(emissions_subsector, 'Electricity and heat production') + # SECTOR(ipcc_crt_code_agg, '1.A.1', '1.A.2') # + # Unknown scheme or value raises; a valid value with no labelled nodes + # returns an empty array. def SECTOR(*keys) - scope.energy_sector_nodes(keys) + if keys.size <= 1 + scope.energy_sector_nodes(keys) + else + scope.energy_sector_node_map(keys.first, keys.drop(1)) + end end - # Returns an Array of {Qernel::Node} for given molecule sector. See SECTOR. + # Returns an Array of {Qernel::Node} for a molecule sector. See SECTOR; + # identical dispatch, normalization and error contract, resolved against + # the molecule graph. def MSECTOR(*keys) - scope.molecule_sector_nodes(keys) + if keys.size <= 1 + scope.molecule_sector_nodes(keys) + else + scope.molecule_sector_map(keys.first, keys.drop(1)) + end end # Returns an Array with {Qernel::Node} for given energy use. # - # See Qernel::Node::USES - # # Examples # # USE(energetic) @@ -145,6 +160,11 @@ def USE(*keys) scope.energy_use_nodes(keys) end + # Returns an Array of {Qernel::Node} for given molecule use. See USE. + def MUSE(*keys) + scope.molecule_use_nodes(keys) + end + # Returns an Array of {Qernel::Carrier} for given key(s). Returns carriers belonging to the # energy graph. # @@ -321,16 +341,17 @@ def DATASET_CURVE(key) # Returns an attribute {Qernel::Emissions} or {Qernel::Emissions::ScopedSector} or a value. # # Emissions data is loaded from CSV files in ETSource with the following structure: - # etm_sector, etm_subsector, use, ghg, unit, value + # etm_sector, etm_subsector, use, ghg, year, unit, value # # Parameters: # - sector: ETM sector name (e.g., 'households', 'buildings_non_specified') # Dashes/dots in sector names are converted to underscores for key generation # - use: Emission use type (energetic, non_energetic) - REQUIRED when accessing values # - ghg: GHG type (co2, other_ghg) - optional - # - year: Year of emission (e.g., 1990) - optional, reads from emissions_YEAR.csv files + # - year: Pass 1990 to read the historic baseline; the present year is + # implicit and needs no argument # - # Key generation combines: sector_[subsector_]use_ghg[_year] + # Key generation combines: sector_[subsector_]use_ghg[_1990] # Note: Unit column from CSV is not included in keys, blank values return nil # # EMISSIONS() without any keys returns {Qernel::Emissions} @@ -351,21 +372,50 @@ def DATASET_CURVE(key) # EMISSIONS(sector, use, ghg) or EMISSIONS(sector, use, ghg, year) returns an emission value # # Examples - # EMISSIONS(households, energetic, other_ghg) # => 12.0 (from emissions.csv) - # EMISSIONS(households, energetic, co2, 1990) # => value (from emissions_1990.csv) + # EMISSIONS(households, energetic, other_ghg) # => 12.0 (present year) + # EMISSIONS(households, energetic, co2, 1990) # => value for 1990 # EMISSIONS(buildings_non_specified, energetic, other_ghg) # => 18.0 # def EMISSIONS(*keys) return scope.graph.emissions if keys.empty? + # Mapped form: EMISSIONS(scheme, value, ghg[, 1990]). Dispatches on the + # first argument being a known classification scheme, because the legacy + # arities overlap. Read-only: sums the store over the mapping's resolved + # (sector label, use) pairs; never returns a scoped-sector handle. + return mapped_emissions(keys) if scope.sector_resolver.scheme?(keys.first) + # Convert dashes/dots to underscores in the first key (sector) keys[0] = keys.first.to_s.tr('-.', '_').to_sym # EMISSIONS(sector, use) -> return ScopedSector for UPDATE operations return scope.graph.emissions.scope(keys.join('_').to_sym) if keys.size == 2 - # EMISSIONS(sector, use, ghg [, year]) -> return value - scope.graph.emissions[keys.join('_').to_sym] + # EMISSIONS(sector, use, ghg [, year]) -> return value. + # The present year is implicit; only 1990 is suffixed with a year. + scope.graph.emissions.value_for(*keys.first(3), year: keys[3]) + end + + private + + # Internal: The mapped EMISSIONS form. Resolves `scheme`/`value` to + # (sector label, use) pairs and lets the emissions store sum over them. + def mapped_emissions(keys) + if keys.size > 4 + raise Gql::GqlError, "EMISSIONS(#{keys.first.inspect}, ...) accepts a single value: " \ + 'EMISSIONS(scheme, value, ghg[, 1990]).' + end + + scheme, value, ghg, year = keys + + if ghg.nil? + raise Gql::GqlError, "EMISSIONS(#{scheme.inspect}, ...) needs a GHG argument, e.g. " \ + "EMISSIONS(#{scheme.inspect}, #{value.inspect}, co2)" + end + + scope.graph.emissions.sum_pairs( + scope.sector_resolver.pairs(scheme, value), ghg, year: year + ) end end end diff --git a/app/models/qernel/emissions.rb b/app/models/qernel/emissions.rb index e300c2b27..3a04c67f9 100644 --- a/app/models/qernel/emissions.rb +++ b/app/models/qernel/emissions.rb @@ -1,39 +1,58 @@ module Qernel - # Class for getting and setting emissions data + # Class for getting and setting emissions data. # Behaves much like Qernel::Area, can be seen as an extension of - # area attributes, scoped for emissions + # area attributes, scoped for emissions. + # + # == Data Structure # # Emissions data is loaded from CSV files in ETSource with structure: - # etm_sector, etm_subsector, use, ghg, unit, value + # etm_sector, etm_subsector, use, ghg, year, unit, value + # + # Values are stored flat, keyed as: sector_subsector_use_ghg[_1990] + # Examples: + # - buildings_non_specified_energetic_other_ghg + # - energy_electricity_and_heat_production_energetic_co2_1990 + # + # == Year Handling # - # Keys are generated as: sector_[subsector_]use_ghg[_year] - # Example: buildings_non_specified_energetic_co2 + # The present year is implicit and carries no suffix. Only the historic + # 1990 baseline is suffixed with its year, so both can coexist in the + # same dataset. class Emissions include DatasetAttributes dataset_accessors ::Etsource::Dataset.emissions_keys attr_accessor :graph + # Public: The flat store key for the given parts, e.g. (sector, use, ghg). + # The single place the `parts_joined_by_underscores[_1990]` schema lives. + # + # Returns a Symbol. + def self.key_for(*parts, year: nil) + [*parts, (1990 if year.to_i == 1990)].compact.join('_').to_sym + end + # Queryable object that provides scoped access to emissions data # # GQL uses this to scope the sector for easier queries and input/update # statements in etsource # # Example: - # EMISSIONS(households, energetic) returns a ScopedSector + # EMISSIONS(households_non_specified, energetic) returns a ScopedSector # Then UPDATE can call: scoped.co2 = 100.0 class ScopedSector - def initialize(emissions, scope) + def initialize(emissions, scope, year = nil) @emissions = emissions @scope = scope + @year = year end def [](attr_name) - @emissions[scoped_method(attr_name).to_sym] + @emissions[scoped_method(attr_name)] end def []=(attr_name, value) - @emissions[scoped_method(attr_name).to_sym] = value + @emissions[scoped_method(attr_name)] = value end def inspect @@ -41,29 +60,38 @@ def inspect end def scoped_method(method_name) - "#{@scope}_#{method_name}" + Emissions.key_for(@scope, method_name.to_s.delete_suffix('='), year: @year) end def respond_to_missing?(method_name, include_private = false) - data_key = scoped_method(method_name).split('=').first - - @emissions.respond_to?(data_key) || super + if method_name.to_s.end_with?('=') + scope_exists? + else + @emissions.respond_to?(scoped_method(method_name)) || super + end end def method_missing(method_name, *args) - data_key = scoped_method(method_name).split('=').first.to_sym + key = scoped_method(method_name) - # Validate the key exists for both getters and setters - unless @emissions.respond_to?(data_key) - raise NoMethodError, "undefined method `#{method_name}' for #{inspect}" - end + if method_name.to_s.end_with?('=') + unless scope_exists? + raise NoMethodError, "undefined method `#{method_name}' for #{inspect}" + end - if data_key.to_s == scoped_method(method_name) - @emissions[data_key] + @emissions[key] = args.first else - @emissions[data_key] = args.first + @emissions[key] end end + + private + + # Scoped writes work for any region that has the scope. + def scope_exists? + prefix = "#{@scope}_" + @emissions.dataset_attributes.keys.any? { |key| key.to_s.start_with?(prefix) } + end end def initialize(graph = nil) @@ -74,9 +102,24 @@ def initialize(graph = nil) # Public: define the sector scope for access to the hashed emission keys # + # sector - Scope identifier (e.g., :buildings_non_specified_energetic). + # year - Optional year. Pass 1990 to target the historic baseline. + # # Returns a scoped version of the emissions data - def scope(sector) - ScopedSector.new(self, sector) + def scope(sector, year = nil) + ScopedSector.new(self, sector, year) + end + + # Public: The stored value for (sector, use, ghg), or nil when the store + # has no such key. Pass year: 1990 to read the historic baseline. + def value_for(sector, use, ghg, year: nil) + self[self.class.key_for(sector, use, ghg, year: year)] + end + + # Public: Sums the store over (sector, use) pairs for one GHG. Pairs + # without a stored value count as zero. + def sum_pairs(pairs, ghg, year: nil) + pairs.sum { |sector, use| value_for(sector, use, ghg, year: year) || 0.0 } end end end diff --git a/app/models/qernel/graph.rb b/app/models/qernel/graph.rb index 208a3605e..756d70d79 100644 --- a/app/models/qernel/graph.rb +++ b/app/models/qernel/graph.rb @@ -264,6 +264,18 @@ def sectors nodes.map(&:sector_key).uniq.compact end + # Resolver for scheme-based sector mapping queries (SECTOR(scheme, value)). + # + # Memoized per graph instance. + # + # @return [Qernel::Sectors] + def sector_map + @sector_map ||= begin + sectors = Etsource::Sectors.new + Qernel::Sectors.new(self, sectors.mapping, sectors.node_index(@name)) + end + end + # Return all nodes in the given sector. # # @param sector_key [String,Symbol] sector identifier diff --git a/app/models/qernel/node_api/direct_emissions.rb b/app/models/qernel/node_api/direct_emissions.rb index e486c5ffc..deb5c1a32 100644 --- a/app/models/qernel/node_api/direct_emissions.rb +++ b/app/models/qernel/node_api/direct_emissions.rb @@ -183,6 +183,28 @@ def direct_reporting_emissions_total_ghg_emissions end end + # CO2 utilisation at this node. + # + # Fossil CO2 input utilisation. + # + # @return [Float, nil] CO2 utilisation in kg, or nil if node is not in emissions group + def direct_reporting_emissions_co2_utilisation + with_emissions_node do + direct_co2_input_utilisation_fossil + end + end + + # Biogenic CO2 emissions at this node. + # + # Biogenic CO2 emissions from production. + # + # @return [Float, nil] Biogenic CO2 emissions in kg, or nil if node is not in emissions group + def direct_reporting_emissions_biogenic_co2_emissions + with_emissions_node do + direct_co2_output_production_emissions_biogenic + end + end + private # Yields the given block only if the node belongs to the :emissions group. diff --git a/app/models/qernel/node_api/molecule_emissions.rb b/app/models/qernel/node_api/molecule_emissions.rb index ed6ba446e..e8bb48976 100644 --- a/app/models/qernel/node_api/molecule_emissions.rb +++ b/app/models/qernel/node_api/molecule_emissions.rb @@ -75,6 +75,28 @@ def direct_reporting_emissions_total_ghg_emissions end end + # CO2 utilisation at this node. + # + # Fossil CO2 input utilisation. + # + # @return [Float, nil] CO2 utilisation in kg, or nil if node is not in emissions group + def direct_reporting_emissions_co2_utilisation + with_emissions_node do + 0.0 + end + end + + # Biogenic CO2 emissions at this node. + # + # Biogenic CO2 emissions from production. + # + # @return [Float, nil] Biogenic CO2 emissions in kg, or nil if node is not in emissions group + def direct_reporting_emissions_biogenic_co2_emissions + with_emissions_node do + 0.0 + end + end + private # Yields the given block only if the node belongs to the :emissions group. diff --git a/app/models/qernel/sectors.rb b/app/models/qernel/sectors.rb new file mode 100644 index 000000000..465171747 --- /dev/null +++ b/app/models/qernel/sectors.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +module Qernel + # Per-graph-instance resolver for scheme-based sector queries. + # + # Present and future graphs (and the energy and molecule graphs) are separate + # instances holding separate node objects, so each gets its own resolver via + # the graph's `sector_map` accessor. The mapping and the (label, use) -> node + # key index are read once from {Etsource::Sectors}; resolution turns node keys + # into the live nodes of this graph. + # + # Dispatch and arity live in the GQL layer; aggregation lives on + # {Qernel::Emissions}. This class answers two questions: which (label, use) + # pairs, and which live nodes, belong to (scheme, values)? + class Sectors + def initialize(graph, mapping, node_index) + @graph = graph + @mapping = mapping + @node_index = node_index + @node_cache = {} + @pair_cache = {} + end + + # Public: Whether `scheme` names a classification scheme in the mapping. + # Used by the EMISSIONS first-argument dispatch. + def scheme?(scheme) + @mapping.key?(normalize(scheme)) + end + + # Public: The unique (label, use) pairs falling under any of `values` of + # `scheme`. Reads the mapping only, so it is independent of node labelling; + # the EMISSIONS mapped form sums the store over these pairs. + # + # Raises Gql::UnknownSectorSchemeError / Gql::UnknownSectorValueError. + def pairs(scheme, values) + value_list = Array(values) + key = [normalize(scheme), value_list.map { |value| normalize(value) }] + + @pair_cache[key] ||= resolve_pairs(scheme, value_list) + end + + # Public: The live nodes of this graph whose (label, use) pair falls under + # any of `values` of `scheme` (the union). A value which resolves to pairs + # with no labelled nodes contributes nothing. + def lookup(scheme, values) + value_list = Array(values) + key = [normalize(scheme), value_list.map { |value| normalize(value) }] + + @node_cache[key] ||= + pairs(scheme, value_list).flat_map { |pair| nodes_for_pair(pair) }.uniq + end + + # Public: The live nodes of this graph whose (label, use) pair equals + # `pair` (an already-normalized [label, use]). Used by mapping-driven CSV + # exports as well as {#lookup}. + def nodes_for_pair(pair) + Array(@node_index[pair]).filter_map { |key| @graph.node(key) } + end + + private + + def resolve_pairs(scheme, value_list) + value_map = @mapping[normalize(scheme)] + raise Gql::UnknownSectorSchemeError.new(scheme, @mapping.keys) if value_map.nil? + + value_list.flat_map do |value| + value_map[normalize(value)] || + raise(Gql::UnknownSectorValueError.new(scheme, value)) + end.uniq + end + + def normalize(value) + Atlas::SectorMapping.normalize(value) + end + end +end diff --git a/app/serializers/export/configured_csv_serializer.rb b/app/serializers/export/configured_csv_serializer.rb index 2d3e0e4ef..7ade8414e 100644 --- a/app/serializers/export/configured_csv_serializer.rb +++ b/app/serializers/export/configured_csv_serializer.rb @@ -11,10 +11,11 @@ # # - schema: An array of hashes describing each column in the CSV. Each hash should contain a `name` # key and optionally a `type` key. -# - rows: An array of hashes, each matching the schema. Any keys contained in a hash which are not -# present in the schema are ignored. +# - rows: Either an array of hashes, each matching the schema (query-driven mode; any keys not +# present in the schema are ignored), or a hash `{ require: , order_by: +# }` (mapping-driven mode, see below). `order_by` is optional. # -# For example: +# Query-driven example: # # { # schema: [ @@ -36,131 +37,236 @@ # - "future": The value will be the result of evaluating the query in the future. # - "unit": The value will be the unit of the specified query. # - "query": This expands into three columns: `present`, `future` and `unit` for specified query. -# - "node_group": Hidden column. Its value names a node group; the row is expanded into one row per -# node in that group. Requires a `period:` to be set on the serializer. +# +# Mapping-driven mode: `rows: { require: }` selects every (sector label, use) pair +# in the sector mapping (see Atlas::SectorMapping) whose cell in `require`'s column has a value, in +# mapping-file order by default. An optional `order_by: ` instead orders pairs by the +# raw display value of that column (ascending string comparison), regardless of whether the column is +# included in `schema:`; a pair whose `order_by` cell is blank/`-` sorts last, and pairs tied on +# `order_by` keep their relative mapping-file order. Each pair expands to the energy and molecule +# nodes whose own `sector_label` and `use` match the pair AND which belong to the node's `:emissions` +# group (key-sorted). A pair with zero labelled nodes, or whose only matching nodes aren't in the +# `:emissions` group, legally yields zero rows. Mapping-driven mode requires a `period:` (raises +# ArgumentError at construction otherwise) and permits only the two column types below (any other +# type raises UnsupportedColumnTypeError at construction): +# # - "node_attribute": The value will be the result of calling the attribute named by `value:` in the -# schema on node_api for each expanded node. Requires `node_group` column in schema. -# `value:` may be any Ruby expression evaluated on node_api via instance_eval. -# Supports an optional `transform:` Ruby expression evaluated with `value` bound -# to the result of `value:`. For example: +# schema on node_api for each expanded node. `value:` may be any Ruby expression +# evaluated on node_api via instance_eval. A nil result (e.g. a node not in the +# :emissions group) renders as an empty string without evaluating `transform`. +# Otherwise, an optional `transform:` Ruby expression is evaluated with `value` +# bound to the result of `value:`. For example: # transform: "value * 10e-6" # transform: "value ? :other_ghg : :co2" -class Export::ConfiguredCSVSerializer - # Represents the schema for a column in the CSV file. - class Column - attr_reader :name, :type, :label, :value, :transform - - def initialize(name, type, label: name, value: nil, transform: nil) - @name = name - @type = type || 'literal' - @label = label || name - @value = value - @transform = transform +# - "sector_mapping": The value will be the raw display value of the mapping column named by `value:`, +# for the row's pair. A `-`/blank mapping cell renders as an empty string. +# +# An unknown mapping column named by `require:`, `order_by:`, or a `sector_mapping` column's `value:` +# raises Export::ConfiguredCSVSerializer::UnknownMappingColumnError at construction, naming the valid +# columns. +module Export + class ConfiguredCSVSerializer # rubocop:disable Style/Documentation + class UnknownMappingColumnError < StandardError end - end - # Creates a serializer using an ETSource config. - # - # period - optional :present or :future; when set, node_group rows are expanded using that graph - # and node_attribute columns are evaluated against it. Existing present/future/unit column - # types continue to work regardless of this setting. - def initialize(config, gql, period: nil) - @config = config.symbolize_keys - @config[:schema] = @config[:schema].map(&:symbolize_keys) - - @columns = @config[:schema].flat_map { |c| create_columns(c) } - @columns.delete(@node_group_column) if unpack_nodes? - @gql = gql - @period = period - end + class UnsupportedColumnTypeError < StandardError + end - def data - rows = [@columns.map(&:label)] - @config[:rows].each { |row| serialize_row(row) { |csv_row| rows << csv_row } } - rows - end + # Column types which may appear in a mapping-driven schema. + MAPPING_COLUMN_TYPES = %w[node_attribute sector_mapping].freeze + + # Represents the schema for a column in the CSV file. + class Column + attr_reader :name, :type, :label, :value, :transform - def as_csv - CSV.generate do |csv| - data.each { |row| csv << row } + def initialize(name, type, label: name, value: nil, transform: nil) + @name = name + @type = type || 'literal' + @label = label || name + @value = value + @transform = transform && compile_transform(transform) + end + + private + + # Internal: Compiles the `transform:` expression to a lambda once, rather + # than eval-ing the string for every exported cell. `value` is the + # column's evaluated node attribute. + def compile_transform(expression) + eval( + "->(value) { #{expression} }", # ->(value) { (value * 1e-6).round(6) } + binding, __FILE__, __LINE__ - 1 + ) + end end - end - private + # Creates a serializer using an ETSource config. + # + # period - optional :present or :future; required when `rows:` is mapping-driven. + def initialize(config, gql, period: nil) + @config = config.symbolize_keys + @config[:schema] = @config[:schema].map(&:symbolize_keys) + @config[:rows] = @config[:rows].symbolize_keys if @config[:rows].is_a?(Hash) + + @columns = @config[:schema].flat_map { |c| create_columns(c) } + @gql = gql + @period = period + + return unless mapping_driven? + + @membership_scheme = @config[:rows][:require]&.to_sym + @order_scheme = @config[:rows][:order_by]&.to_sym + validate_mapping_config! + end + + def data + rows = [@columns.map(&:label)] - def serialize_row(row) - if unpack_nodes? - group_name = row[@node_group_column.name] - nodes = graph_interface.group_energy_nodes(group_name) + - graph_interface.group_molecule_nodes(group_name) - nodes.each do |node| - yield @columns.map { |column| serialize_node_column(column, row, node) } + if mapping_driven? + serialize_mapping_rows { |row| rows << row } + else + @config[:rows].each { |row| rows << @columns.map { |column| serialize_column(column, row) } } end - else - yield @columns.map { |column| serialize_column(column, row) } + + rows end - end - def serialize_column(column, row) - value = row[column.name] + def as_csv + CSV.generate do |csv| + data.each { |row| csv << row } + end + end - return '' if value.blank? + private - case column.type - when 'future' then @gql.future.subquery(value).to_s - when 'present' then @gql.present.subquery(value).to_s - when 'unit' then Gquery.get(value).unit.to_s - else value + def mapping_driven? + @config[:rows].is_a?(Hash) + end + + def serialize_mapping_rows + eligible_rows.each do |raw_row| + nodes_for_pair(raw_row.pair).each do |node| + yield @columns.map { |column| serialize_mapping_column(column, raw_row, node) } + end + end end - end - def serialize_node_column(column, row, node) - case column.type - when 'node_attribute' - value = node.node_api.instance_eval(column.value) - value = eval(column.transform) if column.transform - value.to_s - else serialize_column(column, row) + # Rows whose `require:` cell has a value, in mapping-file order. When `order_by:` is set, sorted + # by that column's raw value instead (blank/`-` last), with ties broken by mapping-file order. + def eligible_rows + rows = sectors.raw_rows.select { |raw_row| raw_row.cells[@membership_scheme] } + return rows unless @order_scheme + + rows.sort_by.with_index { |raw_row, index| order_key(raw_row, index) } end - end - def unpack_nodes? - node_group_column.present? - end + def order_key(raw_row, index) + cell = raw_row.cells[@order_scheme] + [cell.nil? ? 1 : 0, cell.to_s, index] + end - def node_group_column - @node_group_column ||= @columns.find { |c| c.type == 'node_group' } - end + def serialize_mapping_column(column, raw_row, node) + if column.type == 'node_attribute' + value = node.node_api.instance_eval(column.value) + return '' if value.nil? - def graph_interface - @gql.public_send(@period) - end + value = column.transform.call(value) if column.transform + value.to_s + else # 'sector_mapping'; the only other type validate_mapping_config! admits + raw_row.cells[column.value.to_sym].to_s + end + end + + # The `:emissions`-group nodes of both live graphs whose (sector_label, use) matches `pair`. + def nodes_for_pair(pair) + nodes = [graph_interface.graph, graph_interface.molecules].flat_map do |graph| + graph.sector_map.nodes_for_pair(pair) + end - def create_columns(column) - if column[:type] != 'query' - return Column.new( - column[:name], - column[:type], - label: column[:label], - value: column[:value], - transform: column[:transform] - ) + nodes.select(&:emissions?).sort_by { |node| node.node_api.key.to_s } end - %w[present future unit].map do |subtype| - Column.new( - column[:name], - subtype, - label: column[:"#{subtype}_label"] || default_label_for(subtype, column[:name]) - ) + def graph_interface + @gql.public_send(@period) + end + + def sectors + @sectors ||= Etsource::Sectors.new + end + + def validate_mapping_config! + if @period.nil? + raise ArgumentError, 'Mapping-driven rows require a period: of :present or :future.' + end + + if @membership_scheme.nil? + raise ArgumentError, 'Mapping-driven rows require a require: mapping column.' + end + + unsupported = @columns.reject { |column| MAPPING_COLUMN_TYPES.include?(column.type) } + + if unsupported.any? + raise UnsupportedColumnTypeError, + "Column types #{unsupported.map(&:type).uniq.inspect} are not supported with " \ + "mapping-driven rows. Supported types: #{MAPPING_COLUMN_TYPES.inspect}." + end + + validate_mapping_references! end - end - def default_label_for(subtype, column_name) - if subtype == 'unit' - "#{column_name} Unit" - else - "#{subtype.capitalize} #{column_name}" + def validate_mapping_references! + valid = sectors.mapping.keys + sector_mapping_columns = @columns.select { |column| column.type == 'sector_mapping' } + references = [@membership_scheme, @order_scheme, *sector_mapping_columns.map(&:value)] + + references.compact.each do |reference| + next if valid.include?(reference.to_sym) + + raise UnknownMappingColumnError, + "Unknown sector mapping column #{reference.inspect}. " \ + "Valid columns: #{valid.map(&:inspect).join(', ')}." + end + end + + def serialize_column(column, row) + value = row[column.name] + + return '' if value.blank? + + case column.type + when 'future' then @gql.future.subquery(value).to_s + when 'present' then @gql.present.subquery(value).to_s + when 'unit' then Gquery.get(value).unit.to_s + else value + end + end + + def create_columns(column) + if column[:type] != 'query' + return Column.new( + column[:name], + column[:type], + label: column[:label], + value: column[:value], + transform: column[:transform] + ) + end + + %w[present future unit].map do |subtype| + Column.new( + column[:name], + subtype, + label: column[:"#{subtype}_label"] || default_label_for(subtype, column[:name]) + ) + end + end + + def default_label_for(subtype, column_name) + if subtype == 'unit' + "#{column_name} Unit" + else + "#{subtype.capitalize} #{column_name}" + end end end end diff --git a/spec/controllers/api/v3/exports_controller_spec.rb b/spec/controllers/api/v3/exports_controller_spec.rb index 374da0fa6..c88612a6d 100644 --- a/spec/controllers/api/v3/exports_controller_spec.rb +++ b/spec/controllers/api/v3/exports_controller_spec.rb @@ -78,12 +78,7 @@ end end - describe 'GET direct_emissions_present.csv' do - before do - request.headers.merge!(headers) - get :direct_emissions_present, params: { id: scenario.id }, format: :csv - end - + shared_examples 'a mapping-driven direct emissions export' do it 'is successful' do expect(response).to be_ok end @@ -92,6 +87,40 @@ expect(response.media_type).to eq('text/csv') end + it 'includes the eight legacy columns, byte-identical to the pre-rework export' do + expect(CSV.parse(response.body).first).to eq( + [ + 'Sector', 'Subsector', 'Key', 'GHG', + 'CO2 production [kton CO2-eq]', 'CO2 capture [kton CO2-eq]', + 'Other GHG emissions [kton CO2-eq]', 'Total GHG emissions [kton CO2-eq]' + ] + ) + end + + it 'renders Sector/Subsector as mapping lookups for a labelled, :emissions-group node' do + row = CSV.parse(response.body).find { |r| r[2] == 'm_waste' } + expect(row).to eq(%w[Waste Non-specified m_waste co2 0.0 0.0 0.0 0.0]) + end + + it 'excludes a labelled node whose pair has no value in the require column' do + expect(CSV.parse(response.body).flatten).not_to include('foo') + end + + it 'excludes a labelled node that is not in the :emissions group' do + # `bar`/`baz`/`lft`/`buildings_space_heating_demand` all carry a sector_label/use that + # matches an exported mapping row, but none is in the :emissions node group. + expect(CSV.parse(response.body).flatten).not_to include('bar', 'baz', 'lft', 'buildings_space_heating_demand') + end + end + + describe 'GET direct_emissions_present.csv' do + before do + request.headers.merge!(headers) + get :direct_emissions_present, params: { id: scenario.id }, format: :csv + end + + include_examples 'a mapping-driven direct emissions export' + it 'sets the CSV filename' do expect(response.headers['Content-Disposition']).to include("direct_emissions_present.#{scenario.id}.csv") end @@ -103,13 +132,7 @@ get :direct_emissions_future, params: { id: scenario.id }, format: :csv end - it 'is successful' do - expect(response).to be_ok - end - - it 'sets the content type to text/csv' do - expect(response.media_type).to eq('text/csv') - end + include_examples 'a mapping-driven direct emissions export' it 'sets the CSV filename' do expect(response.headers['Content-Disposition']).to include("direct_emissions_future.#{scenario.id}.csv") diff --git a/spec/fixtures/etsource/config/direct_emissions_csv.yml b/spec/fixtures/etsource/config/direct_emissions_csv.yml index 0a15463be..9abee5d06 100644 --- a/spec/fixtures/etsource/config/direct_emissions_csv.yml +++ b/spec/fixtures/etsource/config/direct_emissions_csv.yml @@ -1,8 +1,10 @@ schema: - - name: Group - type: node_group - name: Sector + type: sector_mapping + value: emissions_sector - name: Subsector + type: sector_mapping + value: emissions_subsector - name: Key type: node_attribute value: key @@ -25,12 +27,11 @@ schema: value: direct_reporting_emissions_other_ghg_emissions transform: "(value * 1e-6).round(6)" - name: Total_GHG_emissions - label: "Total GHG emissions[kton CO2-eq]" + label: "Total GHG emissions [kton CO2-eq]" type: node_attribute value: direct_reporting_emissions_total_ghg_emissions transform: "(value * 1e-6).round(6)" rows: - - Group: emissions_industry_chemicals - Sector: Industry - Subsector: Chemicals + require: emissions_sector + order_by: ipcc_crt_code diff --git a/spec/fixtures/etsource/config/sector_mapping.csv b/spec/fixtures/etsource/config/sector_mapping.csv new file mode 100644 index 000000000..59db0709d --- /dev/null +++ b/spec/fixtures/etsource/config/sector_mapping.csv @@ -0,0 +1,12 @@ +sector_label,use,emissions_sector,emissions_subsector,ipcc_crt_code_agg,ipcc_crt_code,klimaattafel +energy_electricity_and_heat_production,energetic,Energy,Electricity and heat production,1.A.1,1.A.1.a,Elektriciteit +industry_refineries,energetic,Industry,Refineries,1.A.1,1.A.1.b,Industrie +industry_refineries,non_energetic,Industry,Refineries,1.B,1.B.2.a.iv,Industrie +industry_non_specified,energetic,-,-,1.A.2,1.A.2,Industrie +buildings_non_specified,energetic,Buildings,Non-specified,1.A.4.a,1.A.4.a,Gebouwde omgeving +households_non_specified,energetic,Households,Non-specified,1.A.4.b,1.A.4.b,Gebouwde omgeving +agriculture_non_specified,energetic,Agriculture,Non-specified,1.A.4.c,1.A.4.c,Landbouw +agriculture_non_specified,non_energetic,Agriculture,Non-specified,3,3,Landbouw +waste_non_specified,non_energetic,Waste,Non-specified,5,5,Elektriciteit +energy_fugitive_emissions,non_energetic,Energy,Fugitive emissions,1.B,1.B excl. 1.B.2.a.iv,Elektriciteit +other_heating,energetic,Other,Heating,-,-,Mobiliteit diff --git a/spec/fixtures/etsource/datasets/nl/emissions.csv b/spec/fixtures/etsource/datasets/nl/emissions.csv index c4c204046..0d1d7a66d 100644 --- a/spec/fixtures/etsource/datasets/nl/emissions.csv +++ b/spec/fixtures/etsource/datasets/nl/emissions.csv @@ -1,9 +1,19 @@ -etm_sector,etm_subsector,use,ghg,unit,value -Energy,Electricity and heat production,energetic,other_ghg,kg CO2eq,18.0 -Energy,Fugitive emissions,non_energetic,co2,kg CO2eq,20.0 -Households,Non-specified,energetic,other_ghg,kg CO2eq,7.0 -Buildings,Non-specified,energetic,other_ghg,kg CO2eq,2796620.0 -Industry,Non-specified,energetic,other_ghg,kg CO2eq,100.0 -Agriculture,Non-specified,energetic,other_ghg,kg CO2eq,50.0 -Agriculture,Non-specified,non_energetic,co2,kg CO2eq,75.0 -Waste,Non-specified,non_energetic,co2,kg CO2eq,25.0 +etm_sector,etm_subsector,use,ghg,year,unit,value +Energy,Electricity and heat production,energetic,other_ghg,2023,kg CO2eq,18.0 +Energy,Electricity and heat production,energetic,other_ghg,1990,kg CO2eq,25.0 +Energy,Fugitive emissions,non_energetic,co2,2023,kg CO2eq,20.0 +Energy,Fugitive emissions,non_energetic,co2,1990,kg CO2eq,30.0 +Households,Non-specified,energetic,other_ghg,2023,kg CO2eq,7.0 +Households,Non-specified,energetic,other_ghg,1990,kg CO2eq,10.0 +Buildings,Non-specified,energetic,other_ghg,2023,kg CO2eq,2796620.0 +Buildings,Non-specified,energetic,other_ghg,1990,kg CO2eq,3000000.0 +Industry,Non-specified,energetic,other_ghg,2023,kg CO2eq,100.0 +Industry,Non-specified,energetic,other_ghg,1990,kg CO2eq,150.0 +Agriculture,Non-specified,energetic,other_ghg,2023,kg CO2eq,50.0 +Agriculture,Non-specified,energetic,other_ghg,1990,kg CO2eq,75.0 +Agriculture,Non-specified,non_energetic,other_ghg,2023,kg CO2eq,100.0 +Agriculture,Non-specified,non_energetic,other_ghg,1990,kg CO2eq,125.0 +Agriculture,Non-specified,non_energetic,co2,2023,kg CO2eq,75.0 +Agriculture,Non-specified,non_energetic,co2,1990,kg CO2eq,100.0 +Waste,Non-specified,non_energetic,co2,2023,kg CO2eq,25.0 +Waste,Non-specified,non_energetic,co2,1990,kg CO2eq,35.0 diff --git a/spec/fixtures/etsource/graphs/energy/nodes/buildings/buildings_space_heating_demand.ad b/spec/fixtures/etsource/graphs/energy/nodes/buildings/buildings_space_heating_demand.ad index ef7605b8d..70ed26212 100644 --- a/spec/fixtures/etsource/graphs/energy/nodes/buildings/buildings_space_heating_demand.ad +++ b/spec/fixtures/etsource/graphs/energy/nodes/buildings/buildings_space_heating_demand.ad @@ -6,3 +6,5 @@ - demand = 100.0 - preset_demand = 100.0 - number_of_units = 200.0 +- sector_label = buildings_non_specified +- use = energetic diff --git a/spec/fixtures/etsource/graphs/energy/nodes/nosector/bar.ad b/spec/fixtures/etsource/graphs/energy/nodes/nosector/bar.ad index 1b98c7d08..7d0e985d0 100644 --- a/spec/fixtures/etsource/graphs/energy/nodes/nosector/bar.ad +++ b/spec/fixtures/etsource/graphs/energy/nodes/nosector/bar.ad @@ -1 +1,3 @@ - groups = [application_group] +- sector_label = industry_refineries +- use = energetic diff --git a/spec/fixtures/etsource/graphs/energy/nodes/nosector/baz.ad b/spec/fixtures/etsource/graphs/energy/nodes/nosector/baz.ad index d2f2e56cb..150b40384 100644 --- a/spec/fixtures/etsource/graphs/energy/nodes/nosector/baz.ad +++ b/spec/fixtures/etsource/graphs/energy/nodes/nosector/baz.ad @@ -1,2 +1,4 @@ - groups = [] - graph_methods = [demand_setter] +- sector_label = industry_refineries +- use = non_energetic diff --git a/spec/fixtures/etsource/graphs/energy/nodes/nosector/foo.ad b/spec/fixtures/etsource/graphs/energy/nodes/nosector/foo.ad index 636979b5d..e158cc9e9 100644 --- a/spec/fixtures/etsource/graphs/energy/nodes/nosector/foo.ad +++ b/spec/fixtures/etsource/graphs/energy/nodes/nosector/foo.ad @@ -3,3 +3,5 @@ - merit_order.type = dispatchable - merit_order.group = solar_pv - graph_methods = [demand_setter] +- sector_label = industry_non_specified +- use = energetic diff --git a/spec/fixtures/etsource/graphs/energy/nodes/nosector/lft.ad b/spec/fixtures/etsource/graphs/energy/nodes/nosector/lft.ad index 27c3b9df6..1254ea0ec 100644 --- a/spec/fixtures/etsource/graphs/energy/nodes/nosector/lft.ad +++ b/spec/fixtures/etsource/graphs/energy/nodes/nosector/lft.ad @@ -1,2 +1,6 @@ - groups = [] - preset_demand = 0.0 + +# Sector mapping test assignment (blank-cell case: "-" ipcc, reachable via klimaattafel) +- sector_label = other_heating +- use = energetic diff --git a/spec/fixtures/etsource/graphs/molecules/nodes/m_right_one.ad b/spec/fixtures/etsource/graphs/molecules/nodes/m_right_one.ad index e9fdc0737..b2102dcac 100644 --- a/spec/fixtures/etsource/graphs/molecules/nodes/m_right_one.ad +++ b/spec/fixtures/etsource/graphs/molecules/nodes/m_right_one.ad @@ -1,2 +1,3 @@ - output.co2 = 1.0 +- use = energetic ~ demand = 0.75 diff --git a/spec/fixtures/etsource/graphs/molecules/nodes/m_right_two.ad b/spec/fixtures/etsource/graphs/molecules/nodes/m_right_two.ad index 9067dc98a..211a9f6fa 100644 --- a/spec/fixtures/etsource/graphs/molecules/nodes/m_right_two.ad +++ b/spec/fixtures/etsource/graphs/molecules/nodes/m_right_two.ad @@ -1,2 +1,3 @@ - output.co2 = 1.0 +- use = non_energetic ~ demand = 0.25 diff --git a/spec/fixtures/etsource/graphs/molecules/nodes/nosector/m_waste.ad b/spec/fixtures/etsource/graphs/molecules/nodes/nosector/m_waste.ad new file mode 100644 index 000000000..1bf13b204 --- /dev/null +++ b/spec/fixtures/etsource/graphs/molecules/nodes/nosector/m_waste.ad @@ -0,0 +1,3 @@ +- groups = [emissions] +- sector_label = waste_non_specified +- use = non_energetic diff --git a/spec/models/etsource/dataset/import_spec.rb b/spec/models/etsource/dataset/import_spec.rb new file mode 100644 index 000000000..c9e9db180 --- /dev/null +++ b/spec/models/etsource/dataset/import_spec.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Etsource::Dataset::Import, :etsource_fixture do + describe '#load_emission_data' do + let(:emissions) { described_class.new('nl').send(:load_emission_data) } + + it 'loads flat emission values keyed by joined CSV columns' do + expect(emissions[:emissions_data][:energy_fugitive_emissions_non_energetic_co2]) + .to eq(20.0) + end + end +end diff --git a/spec/models/gql/command_spec.rb b/spec/models/gql/command_spec.rb new file mode 100644 index 000000000..8b3cd4c19 --- /dev/null +++ b/spec/models/gql/command_spec.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +require 'spec_helper' + +module Gql + describe Command do + describe 'cleaning the source' do + it 'strips whitespace outside string literals' do + expect(described_class.new("SUM(\n 1,\t2\n)").source).to eq('SUM(1,2)') + end + + it 'preserves whitespace inside single-quoted strings' do + expect(described_class.new("SECTOR(emissions_subsector, 'Fuels production')").source) + .to eq("SECTOR(emissions_subsector,'Fuels production')") + end + + it 'preserves whitespace inside double-quoted strings' do + expect(described_class.new('V(foo, "demand * conversion")').source) + .to eq('V(foo,"demand * conversion")') + end + + it 'removes a label prefix' do + expect(described_class.new('future:SUM(1, 2)').source).to eq('SUM(1,2)') + end + + it 'does not remove label-like text inside a string literal' do + expect(described_class.new("'future:keep me'").source).to eq("'future:keep me'") + end + + it 'cleans a nil source to an empty string' do + expect(described_class.new(nil).source).to eq('') + end + end + end +end diff --git a/spec/models/gql/runtime/functions/lookup_spec.rb b/spec/models/gql/runtime/functions/lookup_spec.rb index 60fc6ec9e..2c65f1eae 100644 --- a/spec/models/gql/runtime/functions/lookup_spec.rb +++ b/spec/models/gql/runtime/functions/lookup_spec.rb @@ -3,6 +3,7 @@ module Gql::Runtime::Functions describe Lookup, :etsource_fixture do let(:gql) { Scenario.default.gql(prepare: true) } + let(:molecule_graph) { gql.future.molecules } let(:result) do |example| gql.query_future(example.metadata[:example_group][:description]) @@ -51,21 +52,236 @@ module Gql::Runtime::Functions end describe "EMISSIONS(buildings_non_specified, energetic, other_ghg)" do - it 'returns the emission value' do + it 'returns the present-year emission value' do expect(result).to eq(2796620.0) end end describe "EMISSIONS(energy_electricity_and_heat_production, energetic, other_ghg)" do - it 'returns the emission value' do + it 'returns the present-year emission value' do expect(result).to eq(18.0) end end describe 'EMISSIONS(households_non_specified, energetic, other_ghg)' do - it 'returns the emission value' do + it 'returns the present-year emission value' do expect(result).to eq(7.0) end end + + describe 'EMISSIONS(households_non_specified, energetic, other_ghg, 1990)' do + it 'returns the 1990 baseline emission value' do + expect(result).to eq(10.0) + end + end + + # MUSE + # ---- + + describe 'MUSE(energetic)' do + it 'returns [Node(m_right_one)]' do + expect(result).to eq([molecule_graph.node(:m_right_one)]) + end + end + + describe 'MUSE(non_energetic)' do + it 'returns [Node(m_right_two), Node(m_waste)]' do + expect(result).to eq([ + molecule_graph.node(:m_right_two), + molecule_graph.node(:m_waste) + ]) + end + end + + describe 'MUSE(energetic, non_energetic)' do + it 'returns [Node(m_right_one), Node(m_right_two), Node(m_waste)]' do + expect(result).to eq([ + molecule_graph.node(:m_right_one), + molecule_graph.node(:m_right_two), + molecule_graph.node(:m_waste) + ]) + end + end + + describe 'MUSE(undefined)' do + it 'returns []' do + expect(result).to eq([]) + end + end + + # SECTOR - scheme-based mapping lookups (energy graph) + # --------------------------------------------------- + + def keys_of(nodes) + nodes.map(&:key).sort + end + + describe 'legacy one-argument SECTOR' do + it 'is unchanged: returns the namespace-sector nodes' do + legacy = gql.query_future('SECTOR(nosector)') + direct = gql.future_graph.sector_nodes(:nosector) + + expect(keys_of(legacy)).to eq(keys_of(direct)) + end + end + + describe "SECTOR(klimaattafel, 'Industrie')" do + it 'returns the labelled energy nodes in that klimaattafel' do + expect(keys_of(result)).to eq(%i[bar baz foo]) + end + end + + describe "SECTOR(ipcc_crt_code_agg, '1.A.1')" do + it 'returns only the energetic-use node for that IPCC category' do + expect(keys_of(result)).to eq(%i[bar]) + end + end + + describe "SECTOR(ipcc_crt_code_agg, '1.B')" do + it 'returns only the non-energetic-use node (pair correctness)' do + expect(keys_of(result)).to eq(%i[baz]) + end + end + + describe "SECTOR(ipcc_crt_code_agg, '1.A.1', '1.A.2')" do + it 'returns the union over several values' do + expect(keys_of(result)).to eq(%i[bar foo]) + end + end + + describe 'SECTOR(sector_label, industry_refineries)' do + it 'resolves the canonical key directly (both uses)' do + expect(keys_of(result)).to eq(%i[bar baz]) + end + end + + describe "SECTOR(klimaattafel, 'Gebouwde omgeving')" do + it 'resolves a quoted value containing a space' do + expect(keys_of(result)).to eq(%i[buildings_space_heating_demand]) + end + end + + describe "SECTOR(klimaattafel, 'Landbouw')" do + it 'returns [] for a valid value with no labelled nodes' do + expect(result).to eq([]) + end + end + + describe "SECTOR(impossible, 'x')" do + it 'raises naming the valid schemes' do + expect { result }.to raise_error(/Valid schemes.*klimaattafel/m) + end + end + + describe "SECTOR(klimaattafel, 'Nonexistent')" do + it 'raises for an unknown value' do + expect { result }.to raise_error(/Unknown value "Nonexistent"/) + end + end + + # Blank / "-" cells. The `other_heating,energetic` row has a "-" ipcc cell + # (fixture) and the `lft` node is labelled to it. + + describe "SECTOR(klimaattafel, 'Mobiliteit')" do + it 'reaches a "-"-celled row through a populated scheme' do + expect(keys_of(result)).to eq(%i[lft]) + end + end + + describe 'SECTOR(sector_label, other_heating)' do + it 'reaches the "-"-celled row by its canonical key' do + expect(keys_of(result)).to eq(%i[lft]) + end + end + + describe "SECTOR(ipcc_crt_code_agg, '-')" do + it 'raises: a "-" cell is unqueryable, not a category' do + expect { result }.to raise_error(/Unknown value/) + end + end + + describe "EMISSIONS(ipcc_crt_code_agg, '-', co2)" do + it 'raises: the mapped form rejects a blank value too' do + expect { result }.to raise_error(/Unknown value/) + end + end + + # MSECTOR - scheme-based mapping lookups (molecule graph) + # ------------------------------------------------------ + + describe "MSECTOR(ipcc_crt_code_agg, '5')" do + it 'returns the labelled molecule node' do + expect(keys_of(result)).to eq(%i[m_waste]) + end + end + + describe "MSECTOR(klimaattafel, 'Elektriciteit')" do + it 'returns only molecule nodes (disjoint from the energy graph)' do + expect(keys_of(result)).to eq(%i[m_waste]) + end + end + + describe "SECTOR(ipcc_crt_code_agg, '5')" do + it 'returns [] on the energy graph for a molecule-only pair' do + expect(result).to eq([]) + end + end + + describe 'legacy one-argument MSECTOR' do + it 'is unchanged: routes to the namespace-sector molecule filter' do + expect(gql.query_future('MSECTOR(anything)')) + .to eq(molecule_graph.sector_nodes(:anything)) + end + end + + # EMISSIONS - mapped scheme form + # ------------------------------ + + describe "EMISSIONS(ipcc_crt_code_agg, '1.A.1', other_ghg)" do + it 'sums the store over the resolved pairs (missing keys as zero)' do + # energy_electricity_and_heat_production_energetic_other_ghg = 18.0 + # industry_refineries_energetic_other_ghg is absent -> 0.0 + expect(result).to eq(18.0) + end + end + + describe "EMISSIONS(ipcc_crt_code_agg, '1.A.1', other_ghg, 1990)" do + it 'reads the 1990 baseline keys' do + expect(result).to eq(25.0) + end + end + + describe "EMISSIONS(ipcc_crt_code_agg, '3', co2)" do + it 'sums only the non-energetic rows the mapping lists' do + # agriculture_non_specified_non_energetic_co2 = 75.0; the energetic + # agriculture row (also Landbouw) is not under IPCC 3. + expect(result).to eq(75.0) + end + end + + describe "EMISSIONS(klimaattafel, 'Landbouw', other_ghg)" do + it 'unions energetic and non-energetic rows of the value' do + # agriculture energetic (50.0) + agriculture non_energetic (100.0) + expect(result).to eq(150.0) + end + end + + describe "EMISSIONS(ipcc_crt_code_agg, '1.A.1')" do + it 'raises without a GHG argument' do + expect { result }.to raise_error(Gql::GqlError, /needs a GHG argument/) + end + end + + describe "EMISSIONS(ipcc_crt_code_agg, '1.A.1', '3', co2, 1990)" do + it 'raises on a multi-value call instead of misreading a value as the GHG' do + expect { result }.to raise_error(Gql::GqlError, /single value/) + end + end + + it 'rejects UPDATE on a mapped EMISSIONS expression' do + expect do + gql.query_future("UPDATE(EMISSIONS(ipcc_crt_code_agg, '1.A.1', co2), co2, 1.0)") + end.to raise_error(StandardError) + end end end diff --git a/spec/models/gql/runtime/functions/update_spec.rb b/spec/models/gql/runtime/functions/update_spec.rb index aace84d43..09d848d27 100644 --- a/spec/models/gql/runtime/functions/update_spec.rb +++ b/spec/models/gql/runtime/functions/update_spec.rb @@ -69,13 +69,6 @@ end end - describe 'UPDATE(EMISSIONS(invalid_sector, energetic), co2, 100.0)' do - it 'raises an error when the emission key does not exist in the dataset' do - # The full key 'invalid_sector_energetic_co2' does not exist in the dataset - expect { result }.to raise_error(Gql::CommandError) - end - end - describe 'UPDATE(EMISSIONS(energy_fugitive_emissions, non_energetic), co2, 0.0)' do before { result } diff --git a/spec/models/qernel/emissions_spec.rb b/spec/models/qernel/emissions_spec.rb index dc696a88b..3d29bcc2f 100644 --- a/spec/models/qernel/emissions_spec.rb +++ b/spec/models/qernel/emissions_spec.rb @@ -2,6 +2,16 @@ module Qernel describe Emissions do + let(:dataset) do + Qernel::Dataset.new(1).tap do |ds| + ds.data[:emissions] = { emissions_data: data } + end + end + + let(:graph) { double('Graph', dataset: dataset) } + let(:emissions) { Emissions.new(graph).tap(&:assign_dataset_attributes) } + let(:data) { {} } + describe '#initialize' do context 'with a graph' do let(:graph) { Qernel::Graph.new } @@ -30,22 +40,101 @@ module Qernel end end - describe '#scope' do - let(:emissions) { Emissions.new.with({}) } + describe '.key_for' do + it 'joins the parts with underscores' do + expect(described_class.key_for(:households, :energetic, :co2)) + .to eq(:households_energetic_co2) + end + + it 'suffixes 1990 for the historic baseline' do + expect(described_class.key_for(:households, :energetic, :co2, year: 1990)) + .to eq(:households_energetic_co2_1990) + end + + it 'ignores any other year (the present year is implicit)' do + expect(described_class.key_for(:households, :energetic, :co2, year: 2019)) + .to eq(:households_energetic_co2) + end + end + + describe '#value_for' do + before do + emissions[:households_non_specified_energetic_co2] = 40.0 + emissions[:households_non_specified_energetic_co2_1990] = 60.0 + end + + it 'reads the present-year value' do + expect(emissions.value_for(:households_non_specified, :energetic, :co2)).to eq(40.0) + end + + it 'reads the 1990 baseline when asked' do + expect(emissions.value_for(:households_non_specified, :energetic, :co2, year: 1990)) + .to eq(60.0) + end + + it 'returns nil for a missing key' do + expect(emissions.value_for(:industry_steel, :energetic, :co2)).to be_nil + end + end + + describe '#sum_pairs' do + before do + emissions[:households_non_specified_energetic_co2] = 40.0 + emissions[:agriculture_non_specified_energetic_co2] = 2.0 + end + it 'sums the stored values over the pairs' do + pairs = [%i[households_non_specified energetic], %i[agriculture_non_specified energetic]] + expect(emissions.sum_pairs(pairs, :co2)).to eq(42.0) + end + + it 'counts pairs without a stored value as zero' do + pairs = [%i[households_non_specified energetic], %i[industry_steel energetic]] + expect(emissions.sum_pairs(pairs, :co2)).to eq(40.0) + end + end + + describe '#scope' do it 'returns a ScopedSector instance' do - scoped = emissions.scope(:households_energetic) + scoped = emissions.scope(:households_non_specified_energetic) expect(scoped).to be_a(Emissions::ScopedSector) end it 'sets the correct scope' do - scoped = emissions.scope(:agriculture_energetic) - expect(scoped.instance_variable_get(:@scope)).to eq(:agriculture_energetic) + scoped = emissions.scope(:agriculture_non_specified_energetic) + expect(scoped.instance_variable_get(:@scope)).to eq(:agriculture_non_specified_energetic) + end + end + + describe 'scope validation' do + context 'with valid scope' do + it 'allows creating a ScopedSector instance' do + expect { emissions.scope(:households_non_specified_energetic) }.not_to raise_error + end + end + + context 'with invalid scope (does not exist in dataset)' do + it 'allows creating the ScopedSector instance' do + # Note: Validation happens on setter, not on scope() call + expect { emissions.scope(:invalid_sector_energetic) }.not_to raise_error + end + + it 'raises NoMethodError when trying to set a value' do + invalid_scope = emissions.scope(:invalid_sector_energetic) + expect { invalid_scope.co2 = 100.0 }.to raise_error( + NoMethodError, + /undefined method `co2=' for / + ) + end + + it 'returns nil when trying to get a value (no validation on getter)' do + invalid_scope = emissions.scope(:invalid_sector_energetic) + expect(invalid_scope.co2).to be_nil + end end end describe Emissions::ScopedSector do - let(:emissions) { Emissions.new.with({}) } let(:scoped) { emissions.scope(:households_non_specified_energetic) } before do @@ -53,64 +142,74 @@ module Qernel emissions[:agriculture_non_specified_energetic_other_ghg] = 25.0 end - describe '#method_missing' do - it 'delegates getter methods to emissions with scoped prefix' do + describe 'GHG accessors' do + it 'reads values with the scoped prefix' do expect(scoped.other_ghg).to eq(50.0) end - it 'delegates getter for a different scope' do + it 'reads values for a different scope' do other_scoped = emissions.scope(:agriculture_non_specified_energetic) expect(other_scoped.other_ghg).to eq(25.0) end - it 'delegates setter methods to emissions with scoped prefix' do + it 'writes values with the scoped prefix' do scoped.other_ghg = 75.0 - # Setters convert to symbol keys via dataset_set expect(emissions.dataset_get(:households_non_specified_energetic_other_ghg)).to eq(75.0) end - it 'delegates setter for a different scope' do + it 'writes values for a different scope' do other_scoped = emissions.scope(:agriculture_non_specified_energetic) other_scoped.other_ghg = 30.0 expect(emissions.dataset_get(:agriculture_non_specified_energetic_other_ghg)).to eq(30.0) end - it 'raises NoMethodError for undefined getter methods' do - expect { scoped.nonexistent_attribute }.to raise_error(NoMethodError) - expect { scoped.invalid_emission }.to raise_error(NoMethodError) - end - - it 'raises NoMethodError for setter keys that do not exist in the dataset' do - expect { scoped.arbitrary_key = 100.0 }.to raise_error(NoMethodError) - expect { scoped.custom_emission_type = 200.0 }.to raise_error(NoMethodError) - expect { scoped.nonexistent = 300.0 }.to raise_error(NoMethodError) + it 'returns nil when the scope has no value for the GHG' do + expect(scoped.co2).to be_nil end - it 'allows setters for emission keys that exist in the dataset' do - expect { scoped.other_ghg = 2.0 }.not_to raise_error - expect(emissions.dataset_get(:households_non_specified_energetic_other_ghg)).to eq(2.0) + it 'allows setting a GHG which has no value yet (runtime UPDATE values)' do + scoped.co2 = 300.0 + expect(emissions.dataset_get(:households_non_specified_energetic_co2)).to eq(300.0) end end - describe '#respond_to_missing?' do - it 'returns true for valid GHG types that exist' do - expect(scoped.respond_to?(:other_ghg)).to be true - end - - it 'returns true for setter methods where the key exists in dataset' do - expect(scoped.respond_to?(:other_ghg=)).to be true - end + describe 'year targeting' do + it 'reads and writes the requested year' do + scoped_1990 = emissions.scope(:households_non_specified_energetic, 1990) + scoped_1990.other_ghg = 12.0 - it 'returns false for setter methods where the key does not exist in dataset' do - expect(scoped.respond_to?(:invalid_key=)).to be false - expect(scoped.respond_to?(:co2=)).to be false # co2 doesn't exist for this scope - expect(scoped.respond_to?(:arbitrary_name=)).to be false + expect(emissions.dataset_get(:households_non_specified_energetic_other_ghg_1990)).to eq(12.0) + expect(scoped_1990.other_ghg).to eq(12.0) + expect(scoped.other_ghg).to eq(50.0) end + end - it 'returns false for getter methods where the key does not exist in dataset' do - expect(scoped.respond_to?(:invalid_key)).to be false - expect(scoped.respond_to?(:co2)).to be false # co2 doesn't exist for this scope - expect(scoped.respond_to?(:nonexistent_attribute)).to be false + describe 'scope existence validation' do + context 'with scope that does not exist in dataset' do + before do + # Create a scope that has no matching keys in dataset + @invalid_scoped = emissions.scope(:nonexistent_sector_energetic) + end + + it 'raises NoMethodError when setting any GHG value' do + expect { @invalid_scoped.co2 = 100.0 }.to raise_error(NoMethodError) + expect { @invalid_scoped.other_ghg = 50.0 }.to raise_error(NoMethodError) + end + + it 'returns nil when getting any GHG value (no validation)' do + expect(@invalid_scoped.co2).to be_nil + expect(@invalid_scoped.other_ghg).to be_nil + end + + it 'does not respond to setter methods' do + expect(@invalid_scoped.respond_to?(:co2=)).to be false + expect(@invalid_scoped.respond_to?(:other_ghg=)).to be false + end + + it 'does not respond to getter methods' do + expect(@invalid_scoped.respond_to?(:co2)).to be false + expect(@invalid_scoped.respond_to?(:other_ghg)).to be false + end end end @@ -141,8 +240,7 @@ module Qernel describe '#inspect' do it 'returns a readable string representation' do - scoped = emissions.scope(:households_energetic) - expect(scoped.inspect).to eq('') + expect(scoped.inspect).to eq('') end end @@ -153,6 +251,8 @@ module Qernel emissions[:energy_fugitive_emissions_non_energetic_co2] = 0.0 emissions[:energy_electricity_and_heat_production_energetic_other_ghg] = 0.0 emissions[:buildings_non_specified_energetic_other_ghg] = 0.0 + emissions[:agriculture_non_specified_energetic_other_ghg] = 0.0 + emissions[:agriculture_non_specified_non_energetic_co2] = 0.0 end it 'handles zero values' do @@ -178,7 +278,6 @@ module Qernel scoped.co2 = 100.0 expect(emissions.dataset_get(:energy_fugitive_emissions_non_energetic_co2)).to eq(100.0) - # Test agriculture keys that actually exist in default dataset ag_energetic = emissions.scope(:agriculture_non_specified_energetic) ag_energetic.other_ghg = 200.0 expect(emissions.dataset_get(:agriculture_non_specified_energetic_other_ghg)).to eq(200.0) diff --git a/spec/models/qernel/node_api/direct_emissions_spec.rb b/spec/models/qernel/node_api/direct_emissions_spec.rb index fc0c06c5d..1d6b0a8c5 100644 --- a/spec/models/qernel/node_api/direct_emissions_spec.rb +++ b/spec/models/qernel/node_api/direct_emissions_spec.rb @@ -2462,4 +2462,193 @@ end end end + + describe '#direct_reporting_emissions_co2_utilisation' do + context 'with fossil CO2 input utilisation' do + # Create a graph: + # [CO2 Source] -> [CO2 Utilisation Plant] -> [Terminus] + # Plant uses fossil CO2 as input for production + let(:builder) do + TestGraphBuilder.new.tap do |builder| + builder.add(:terminus, demand: 100) + builder.add(:co2_utilisation_plant, groups: [:emissions], co2_utilisation_per_mj: 0.08) + builder.add(:co2_source, groups: [:primary_energy_demand]) + + builder.connect(:co2_source, :co2_utilisation_plant, :co2, type: :share) + builder.connect(:co2_utilisation_plant, :terminus, :electricity, type: :share) + + builder.carrier_attrs(:co2, co2_conversion_per_mj: 0.0) + builder.carrier_attrs(:electricity, co2_conversion_per_mj: 0.0) + end + end + + let(:graph) { builder.to_qernel } + let(:co2_plant) { graph.node(:co2_utilisation_plant) } + + it 'reports CO2 utilisation equal to fossil input utilisation' do + # Total useful output: 100 MJ + # Fossil CO2 utilisation: 100 MJ * 0.08 kg/MJ = 8.0 kg CO2 + # CO2 Utilisation (reporting) = 8.0 kg CO2 + expect(co2_plant).to have_query_value(:direct_reporting_emissions_co2_utilisation, 8.0) + end + + it 'equals direct_co2_input_utilisation_fossil' do + fossil_util = co2_plant.query.direct_co2_input_utilisation_fossil + utilisation = co2_plant.query.direct_reporting_emissions_co2_utilisation + + expect(utilisation).to eq(fossil_util) + end + end + + context 'with zero fossil CO2 utilisation' do + # Create a graph: + # [Producer] -> [Plant without CO2 utilisation] -> [Terminus] + # Plant has no CO2 utilisation attribute + let(:builder) do + TestGraphBuilder.new.tap do |builder| + builder.add(:terminus, demand: 100) + builder.add(:regular_plant, groups: [:emissions]) + builder.add(:producer, groups: [:primary_energy_demand]) + + builder.connect(:producer, :regular_plant, :natural_gas, type: :share) + builder.connect(:regular_plant, :terminus, :electricity, type: :share) + + builder.carrier_attrs(:natural_gas, co2_conversion_per_mj: 0.0564) + builder.carrier_attrs(:electricity, co2_conversion_per_mj: 0.0) + end + end + + let(:graph) { builder.to_qernel } + let(:regular_plant) { graph.node(:regular_plant) } + + it 'reports zero CO2 utilisation' do + # No CO2 utilisation = 0 kg CO2 + expect(regular_plant).to have_query_value(:direct_reporting_emissions_co2_utilisation, 0.0) + end + + it 'equals direct_co2_input_utilisation_fossil (zero)' do + fossil_util = regular_plant.query.direct_co2_input_utilisation_fossil + utilisation = regular_plant.query.direct_reporting_emissions_co2_utilisation + + expect(utilisation).to eq(fossil_util) + expect(utilisation).to eq(0.0) + end + end + + context 'with a non-emissions node' do + let(:builder) do + TestGraphBuilder.new.tap do |builder| + builder.add(:terminus, demand: 100) + builder.add(:non_emissions_node) + builder.add(:producer, groups: [:primary_energy_demand]) + + builder.connect(:producer, :non_emissions_node, :natural_gas, type: :share) + builder.connect(:non_emissions_node, :terminus, :electricity, type: :share) + + builder.carrier_attrs(:natural_gas, co2_conversion_per_mj: 0.0564) + end + end + + let(:graph) { builder.to_qernel } + let(:non_emissions_node) { graph.node(:non_emissions_node) } + + it 'returns nil for non-emissions nodes' do + expect(non_emissions_node.query.direct_reporting_emissions_co2_utilisation).to be_nil + end + end + end + + describe '#direct_reporting_emissions_biogenic_co2_emissions' do + context 'with biogenic production emissions' do + # Create a graph: + # [Biogenic Waste Producer] -> [Biogenic Plant] -> [Terminus] + # Plant produces biogenic CO2 emissions during production + let(:builder) do + TestGraphBuilder.new.tap do |builder| + builder.add(:terminus, demand: 100) + builder.add(:biogenic_plant, groups: [:emissions]) + builder.add(:biogenic_waste_producer, groups: [:primary_energy_demand]) + + builder.connect(:biogenic_waste_producer, :biogenic_plant, :biogenic_waste, type: :share) + builder.connect(:biogenic_plant, :terminus, :heat, type: :share) + + builder.carrier_attrs(:biogenic_waste, potential_co2_conversion_per_mj: 0.05) + builder.carrier_attrs(:heat, potential_co2_conversion_per_mj: 0.0) + end + end + + let(:graph) { builder.to_qernel } + let(:biogenic_plant) { graph.node(:biogenic_plant) } + + it 'reports biogenic CO2 emissions equal to biogenic production emissions' do + # Biogenic: 100 MJ * 0.05 kg/MJ = 5.0 kg CO2 + # Biogenic CO2 emissions (reporting) = 5.0 kg CO2 + expect(biogenic_plant).to have_query_value(:direct_reporting_emissions_biogenic_co2_emissions, 5.0) + end + + it 'equals direct_co2_output_production_emissions_biogenic' do + biogenic = biogenic_plant.query.direct_co2_output_production_emissions_biogenic + biogenic_reporting = biogenic_plant.query.direct_reporting_emissions_biogenic_co2_emissions + + expect(biogenic_reporting).to eq(biogenic) + end + end + + context 'with zero biogenic emissions' do + # Create a graph: + # [Fossil Producer] -> [Fossil Plant] -> [Terminus] + # Plant has no biogenic inputs + let(:builder) do + TestGraphBuilder.new.tap do |builder| + builder.add(:terminus, demand: 100) + builder.add(:fossil_plant, groups: [:emissions]) + builder.add(:fossil_producer, groups: [:primary_energy_demand]) + + builder.connect(:fossil_producer, :fossil_plant, :natural_gas, type: :share) + builder.connect(:fossil_plant, :terminus, :electricity, type: :share) + + builder.carrier_attrs(:natural_gas, co2_conversion_per_mj: 0.0564) + builder.carrier_attrs(:electricity, co2_conversion_per_mj: 0.0) + end + end + + let(:graph) { builder.to_qernel } + let(:fossil_plant) { graph.node(:fossil_plant) } + + it 'reports zero biogenic CO2 emissions' do + # No biogenic inputs = 0 kg CO2 + expect(fossil_plant).to have_query_value(:direct_reporting_emissions_biogenic_co2_emissions, 0.0) + end + + it 'equals direct_co2_output_production_emissions_biogenic (zero)' do + biogenic = fossil_plant.query.direct_co2_output_production_emissions_biogenic + biogenic_reporting = fossil_plant.query.direct_reporting_emissions_biogenic_co2_emissions + + expect(biogenic_reporting).to eq(biogenic) + expect(biogenic_reporting).to eq(0.0) + end + end + + context 'with a non-emissions node' do + let(:builder) do + TestGraphBuilder.new.tap do |builder| + builder.add(:terminus, demand: 100) + builder.add(:non_emissions_node) + builder.add(:producer, groups: [:primary_energy_demand]) + + builder.connect(:producer, :non_emissions_node, :natural_gas, type: :share) + builder.connect(:non_emissions_node, :terminus, :electricity, type: :share) + + builder.carrier_attrs(:natural_gas, co2_conversion_per_mj: 0.0564) + end + end + + let(:graph) { builder.to_qernel } + let(:non_emissions_node) { graph.node(:non_emissions_node) } + + it 'returns nil for non-emissions nodes' do + expect(non_emissions_node.query.direct_reporting_emissions_biogenic_co2_emissions).to be_nil + end + end + end end diff --git a/spec/models/qernel/node_api/molecule_emissions_spec.rb b/spec/models/qernel/node_api/molecule_emissions_spec.rb index ec8ce1518..f1d6de08c 100644 --- a/spec/models/qernel/node_api/molecule_emissions_spec.rb +++ b/spec/models/qernel/node_api/molecule_emissions_spec.rb @@ -190,6 +190,38 @@ end end + describe '#direct_reporting_emissions_co2_utilisation' do + context 'node in emissions group' do + it 'returns 0.0' do + expect(node.query.direct_reporting_emissions_co2_utilisation).to eq(0.0) + end + end + + context 'node not in emissions group' do + let(:node_groups) { [] } + + it 'returns nil' do + expect(node.query.direct_reporting_emissions_co2_utilisation).to be_nil + end + end + end + + describe '#direct_reporting_emissions_biogenic_co2_emissions' do + context 'node in emissions group' do + it 'returns 0.0' do + expect(node.query.direct_reporting_emissions_biogenic_co2_emissions).to eq(0.0) + end + end + + context 'node not in emissions group' do + let(:node_groups) { [] } + + it 'returns nil' do + expect(node.query.direct_reporting_emissions_biogenic_co2_emissions).to be_nil + end + end + end + describe '#ghg_carrier' do context 'with a co2 input slot' do before do @@ -222,6 +254,8 @@ expect(node.query.direct_reporting_emissions_co2_capture).to be_nil expect(node.query.direct_reporting_emissions_other_ghg_emissions).to be_nil expect(node.query.direct_reporting_emissions_total_ghg_emissions).to be_nil + expect(node.query.direct_reporting_emissions_co2_utilisation).to be_nil + expect(node.query.direct_reporting_emissions_biogenic_co2_emissions).to be_nil end end end diff --git a/spec/serializers/export/configured_csv_serializer_spec.rb b/spec/serializers/export/configured_csv_serializer_spec.rb index 99b0056c0..c858fec43 100644 --- a/spec/serializers/export/configured_csv_serializer_spec.rb +++ b/spec/serializers/export/configured_csv_serializer_spec.rb @@ -122,108 +122,365 @@ end end - context 'when given a "node_group" column' do - let(:node_api_1) { instance_double('Qernel::NodeApi::EnergyApi', key: 'node_a') } - let(:node_api_2) { instance_double('Qernel::NodeApi::EnergyApi', key: 'node_b') } - let(:node_1) { instance_double('Qernel::Node', node_api: node_api_1) } - let(:node_2) { instance_double('Qernel::Node', node_api: node_api_2) } + context 'when given mapping-driven rows' do + # Real pairs from spec/fixtures/etsource/config/sector_mapping.csv, joined against the real + # labelled fixture nodes (spec/fixtures/etsource/graphs). In mapping-file order: + # + # energy_electricity_and_heat_production/energetic -> Energy, emissions_sector set, 0 nodes + # industry_refineries/energetic -> Industry, node `bar` (dual-use label) + # industry_refineries/non_energetic -> Industry, node `baz` (dual-use label) + # industry_non_specified/energetic -> emissions_sector blank -> excluded + # (node `foo` exists but must not appear) + # buildings_non_specified/energetic -> Buildings, node `buildings_space_heating_demand` + # households_non_specified/energetic -> 0 nodes + # agriculture_non_specified/energetic -> 0 nodes + # agriculture_non_specified/non_energetic -> 0 nodes + # waste_non_specified/non_energetic -> Waste, node `m_waste` (molecule graph) + # energy_fugitive_emissions/non_energetic -> 0 nodes + # other_heating/energetic -> Other, node `lft`, blank ipcc cell let(:config) do { schema: [ - { name: 'Group', type: 'node_group' }, - { name: 'Sector' }, - { name: 'Node', type: 'node_attribute', value: 'key' } + { name: 'Sector', type: 'sector_mapping', value: 'emissions_sector' }, + { name: 'Key', type: 'node_attribute', value: 'key' }, + { name: 'IPCC', type: 'sector_mapping', value: 'ipcc_crt_code_agg' } ], - rows: [ - { 'Group' => 'some_group', 'Sector' => 'Industry' } - ] + rows: { require: 'emissions_sector' } } end - let(:node_api_3) { instance_double('Qernel::NodeApi::MoleculeApi', key: 'node_c') } - let(:node_3) { instance_double('Qernel::Node', node_api: node_api_3) } - - let(:gql) { Scenario.default.gql } + let(:gql) { Scenario.default.gql } let(:serializer) { described_class.new(config, gql, period: :future) } - before do - allow(gql.future).to receive(:group_energy_nodes).with('some_group').and_return([node_1, node_2]) - allow(gql.future).to receive(:group_molecule_nodes).with('some_group').and_return([node_3]) + let(:node_bar) do + instance_double( + 'Qernel::Node', + emissions?: true, + node_api: instance_double( + 'Qernel::NodeApi::EnergyApi', key: :bar, direct_reporting_emissions_co2_production: nil + ) + ) end - - it 'excludes the node_group column from the header' do - expect(serializer.data[0]).to eq(%w[Sector Node]) + let(:node_baz) do + instance_double('Qernel::Node', emissions?: true, node_api: instance_double('Qernel::NodeApi::EnergyApi', key: :baz)) + end + let(:node_foo) do + instance_double('Qernel::Node', emissions?: true, node_api: instance_double('Qernel::NodeApi::EnergyApi', key: :foo)) + end + let(:node_buildings) do + instance_double( + 'Qernel::Node', emissions?: true, + node_api: instance_double('Qernel::NodeApi::EnergyApi', key: :buildings_space_heating_demand) + ) + end + let(:node_lft) do + instance_double('Qernel::Node', emissions?: true, node_api: instance_double('Qernel::NodeApi::EnergyApi', key: :lft)) + end + let(:node_waste) do + instance_double('Qernel::Node', emissions?: true, node_api: instance_double('Qernel::NodeApi::MoleculeApi', key: :m_waste)) end - it 'expands the row into one row per node from both graphs' do - expect(serializer.data.length).to eq(4) # header + 2 energy + 1 molecule + let(:energy_nodes) do + { bar: node_bar, baz: node_baz, foo: node_foo, buildings_space_heating_demand: node_buildings, lft: node_lft } end - it 'includes the literal column value on each expanded row' do - expect(serializer.data[1][0]).to eq('Industry') - expect(serializer.data[2][0]).to eq('Industry') - expect(serializer.data[3][0]).to eq('Industry') + let(:molecule_graph) { instance_double('Qernel::Graph') } + + before do + allow(gql.future.graph).to receive(:node) { |key| energy_nodes[key.to_sym] } + allow(gql.future).to receive(:molecules).and_return(molecule_graph) + allow(molecule_graph).to receive(:node).with(:m_waste).and_return(node_waste) + + allow(molecule_graph).to receive(:sector_map) do + sectors = Etsource::Sectors.new + Qernel::Sectors.new(molecule_graph, sectors.mapping, sectors.node_index(:molecules)) + end end - it 'includes energy node_attribute values' do - expect(serializer.data[1][1]).to eq('node_a') - expect(serializer.data[2][1]).to eq('node_b') + it 'includes the CSV headers' do + expect(serializer.data[0]).to eq(%w[Sector Key IPCC]) end - it 'includes molecule node_attribute values' do - expect(serializer.data[3][1]).to eq('node_c') + it 'emits rows in mapping-file order, skipping pairs with zero labelled nodes' do + expect(serializer.data[1..].map { |row| row[1] }).to eq( + %w[bar baz buildings_space_heating_demand m_waste lft] + ) end - context 'with a transform for numeric conversion' do - let(:node_api_1) { instance_double('Qernel::NodeApi::EnergyApi', key: 'node_a', demand: 1_000_000.0) } + it 'excludes a pair whose require column cell is blank, even though it has a labelled node' do + expect(serializer.data.flatten).not_to include('foo') + end + context 'with an "order_by" rule' do let(:config) do { - schema: [ - { name: 'Group', type: 'node_group' }, - { name: 'Demand', type: 'node_attribute', value: 'demand', transform: 'value * 1e-6' } - ], - rows: [{ 'Group' => 'some_group' }] + schema: [{ name: 'Key', type: 'node_attribute', value: 'key' }], + rows: { require: 'emissions_sector', order_by: 'ipcc_crt_code' } } end + it 'orders rows by the raw value of the order_by column, not the mapping-file order' do + # ipcc_crt_code: bar 1.A.1.b, buildings 1.A.4.a, baz 1.B.2.a.iv, m_waste 5, lft blank. + expect(serializer.data[1..].flatten).to eq( + %w[bar buildings_space_heating_demand baz m_waste lft] + ) + end + + it 'sorts a pair with a blank order_by cell last' do + expect(serializer.data.last).to eq(['lft']) + end + end + + context 'when a labelled node is not in the :emissions group' do + before { allow(node_baz).to receive(:emissions?).and_return(false) } + + it 'excludes it, even though its pair matches an exported row' do + expect(serializer.data.flatten).not_to include('baz') + end + + it 'still includes the other node under the same dual-use label' do + expect(serializer.data.flatten).to include('bar') + end + end + + it 'renders the raw display value of the require column (not a slug)' do + expect(serializer.data[1][0]).to eq('Industry') + expect(serializer.data[4][0]).to eq('Waste') + end + + it 'joins a dual-use label by the node\'s own (label, use) pair, not by name alone' do + # `bar` and `baz` share sector_label industry_refineries but differ in `use`, and land under + # different IPCC codes as a result. + expect(serializer.data[1]).to eq(%w[Industry bar 1.A.1]) + expect(serializer.data[2]).to eq(%w[Industry baz 1.B]) + end + + it 'includes molecule-graph nodes alongside energy nodes' do + expect(serializer.data[4]).to eq(%w[Waste m_waste 5]) + end + + it 'renders a raw value with punctuation unchanged' do + expect(serializer.data[1][2]).to eq('1.A.1') + end + + it 'renders a blank mapping cell as an empty string' do + expect(serializer.data[5]).to eq(['Other', 'lft', '']) + end + + context 'with an isolated pair (stubbed Etsource::Sectors)' do + let(:sectors) { instance_double(Etsource::Sectors) } + let(:pair) { %i[industry_refineries energetic] } + let(:raw_row) { Atlas::SectorMapping::RawRow.new(pair, { emissions_sector: 'Industry' }) } + before do - allow(gql.future).to receive(:group_energy_nodes).with('some_group').and_return([node_1]) - allow(gql.future).to receive(:group_molecule_nodes).with('some_group').and_return([]) + allow(Etsource::Sectors).to receive(:new).and_return(sectors) + allow(sectors).to receive(:mapping).and_return({ emissions_sector: {} }) + allow(sectors).to receive(:raw_rows).and_return([raw_row]) + allow(sectors).to receive(:node_index).with(:molecules).and_return({}) + end + + context 'sorting nodes within a pair' do + let(:config) do + { + schema: [{ name: 'Key', type: 'node_attribute', value: 'key' }], + rows: { require: 'emissions_sector' } + } + end + + let(:node_z) do + instance_double('Qernel::Node', emissions?: true, node_api: instance_double('Qernel::NodeApi::EnergyApi', key: :z_node)) + end + let(:node_a) do + instance_double('Qernel::Node', emissions?: true, node_api: instance_double('Qernel::NodeApi::EnergyApi', key: :a_node)) + end + + before do + allow(sectors).to receive(:node_index).with(:energy).and_return({ pair => %i[z_node a_node] }) + allow(gql.future.graph).to receive(:node).with(:z_node).and_return(node_z) + allow(gql.future.graph).to receive(:node).with(:a_node).and_return(node_a) + end + + it 'sorts the expanded nodes by key, regardless of node_index order' do + expect(serializer.data[1..].map { |row| row[0] }).to eq(%w[a_node z_node]) + end + end + + context 'with an "order_by" rule and rows tied on that column' do + let(:pair_a) { %i[pair_a energetic] } + let(:pair_b) { %i[pair_b energetic] } + let(:pair_c) { %i[pair_c energetic] } + + let(:raw_row_a) { Atlas::SectorMapping::RawRow.new(pair_a, { emissions_sector: 'A', ipcc_crt_code: '1' }) } + let(:raw_row_b) { Atlas::SectorMapping::RawRow.new(pair_b, { emissions_sector: 'B', ipcc_crt_code: '1' }) } + let(:raw_row_c) { Atlas::SectorMapping::RawRow.new(pair_c, { emissions_sector: 'C', ipcc_crt_code: '0' }) } + + let(:config) do + { + schema: [{ name: 'Key', type: 'node_attribute', value: 'key' }], + rows: { require: 'emissions_sector', order_by: 'ipcc_crt_code' } + } + end + + let(:node_a) do + instance_double('Qernel::Node', emissions?: true, node_api: instance_double('Qernel::NodeApi::EnergyApi', key: :node_a)) + end + let(:node_b) do + instance_double('Qernel::Node', emissions?: true, node_api: instance_double('Qernel::NodeApi::EnergyApi', key: :node_b)) + end + let(:node_c) do + instance_double('Qernel::Node', emissions?: true, node_api: instance_double('Qernel::NodeApi::EnergyApi', key: :node_c)) + end + + before do + allow(sectors).to receive(:mapping).and_return({ emissions_sector: {}, ipcc_crt_code: {} }) + # File order: A, B, C. A and B tie on ipcc_crt_code ("1"); C has a lower code ("0"). + allow(sectors).to receive(:raw_rows).and_return([raw_row_a, raw_row_b, raw_row_c]) + allow(sectors).to receive(:node_index).with(:energy).and_return( + pair_a => [:node_a], pair_b => [:node_b], pair_c => [:node_c] + ) + allow(gql.future.graph).to receive(:node).with(:node_a).and_return(node_a) + allow(gql.future.graph).to receive(:node).with(:node_b).and_return(node_b) + allow(gql.future.graph).to receive(:node).with(:node_c).and_return(node_c) + end + + it 'sorts by the order_by value first, breaking ties by mapping-file order' do + expect(serializer.data[1..].flatten).to eq(%w[node_c node_a node_b]) + end + end + + context 'when a node_attribute value is nil (node not in the :emissions group)' do + let(:config) do + { + schema: [ + { name: 'Key', type: 'node_attribute', value: 'key' }, + { name: 'CO2', type: 'node_attribute', value: 'direct_reporting_emissions_co2_production', + transform: 'value * 1e-6' } + ], + rows: { require: 'emissions_sector' } + } + end + + let(:node_bar) do + instance_double( + 'Qernel::Node', + emissions?: true, + node_api: instance_double( + 'Qernel::NodeApi::EnergyApi', key: :bar, direct_reporting_emissions_co2_production: nil + ) + ) + end + + before do + allow(sectors).to receive(:node_index).with(:energy).and_return({ pair => [:bar] }) + allow(gql.future.graph).to receive(:node).with(:bar).and_return(node_bar) + end + + it 'renders an empty string without evaluating the transform' do + expect(serializer.data[1]).to eq(['bar', '']) + end + end + end + end + + context 'when a mapping-driven config is invalid' do + let(:gql) { Scenario.default.gql } + let(:mapping_rows) { { require: 'emissions_sector' } } + let(:schema) { [{ name: 'Key', type: 'node_attribute', value: 'key' }] } + + context 'with no period' do + let(:serializer) { described_class.new({ schema: schema, rows: mapping_rows }, gql) } + + it 'raises at construction' do + expect { serializer }.to raise_error(ArgumentError, /period/) + end + end + + context 'with no require: column' do + let(:serializer) do + described_class.new({ schema: schema, rows: { order_by: 'ipcc_crt_code' } }, gql, period: :future) + end + + it 'raises at construction' do + expect { serializer }.to raise_error(ArgumentError, /require/) end + end + + context 'with a column type only valid for query-driven rows' do + let(:schema) { super() + [{ name: 'Total', type: 'query' }] } + let(:serializer) { described_class.new({ schema: schema, rows: mapping_rows }, gql, period: :future) } - it 'applies the transform to the value' do - expect(serializer.data[1][0]).to eq('1.0') + it 'raises at construction instead of rendering silent blank cells' do + expect { serializer }.to raise_error( + Export::ConfiguredCSVSerializer::UnsupportedColumnTypeError, + /node_attribute.*sector_mapping/ + ) end end - context 'with a transform for conditional mapping' do - let(:node_api_1) { double('node_api', key: 'node_a', is_special: true) } - let(:node_api_2) { double('node_api', key: 'node_b', is_special: false) } + context 'with an untyped (literal) column' do + let(:schema) { super() + [{ name: 'Sector' }] } + let(:serializer) { described_class.new({ schema: schema, rows: mapping_rows }, gql, period: :future) } + it 'raises at construction instead of rendering silent blank cells' do + expect { serializer }.to raise_error( + Export::ConfiguredCSVSerializer::UnsupportedColumnTypeError, /literal/ + ) + end + end + end + + context 'when given an unknown sector mapping column' do + let(:gql) { Scenario.default.gql } + let(:serializer) { described_class.new(config, gql, period: :future) } + + context 'as a "rows: require:" reference' do + let(:config) do + { + schema: [{ name: 'Key', type: 'node_attribute', value: 'key' }], + rows: { require: 'impossible' } + } + end + + it 'raises at construction, naming the invalid reference and the valid columns' do + expect { serializer }.to raise_error( + Export::ConfiguredCSVSerializer::UnknownMappingColumnError, + /impossible.*Valid columns.*emissions_sector/m + ) + end + end + + context 'as a "sector_mapping" column value' do let(:config) do { schema: [ - { name: 'Group', type: 'node_group' }, - { name: 'Type', type: 'node_attribute', value: 'is_special', - transform: "value ? 'other_ghg' : 'co2'" } + { name: 'Sector', type: 'sector_mapping', value: 'impossible' } ], - rows: [{ 'Group' => 'some_group' }] + rows: { require: 'emissions_sector' } } end - before do - allow(gql.future).to receive(:group_energy_nodes).with('some_group').and_return([node_1, node_2]) - allow(gql.future).to receive(:group_molecule_nodes).with('some_group').and_return([]) + it 'raises at construction, naming the invalid reference and the valid columns' do + expect { serializer }.to raise_error( + Export::ConfiguredCSVSerializer::UnknownMappingColumnError, + /impossible.*Valid columns.*emissions_sector/m + ) end + end - it 'maps a truthy result via transform' do - expect(serializer.data[1][0]).to eq('other_ghg') + context 'as a "rows: order_by:" reference' do + let(:config) do + { + schema: [{ name: 'Key', type: 'node_attribute', value: 'key' }], + rows: { require: 'emissions_sector', order_by: 'impossible' } + } end - it 'maps a falsy result via transform' do - expect(serializer.data[2][0]).to eq('co2') + it 'raises at construction, naming the invalid reference and the valid columns' do + expect { serializer }.to raise_error( + Export::ConfiguredCSVSerializer::UnknownMappingColumnError, + /impossible.*Valid columns.*emissions_sector/m + ) end end end