From 0155f8988d69a9e846eb165efbfabb8b0b24b410 Mon Sep 17 00:00:00 2001 From: Antti Hukkanen Date: Fri, 15 Nov 2024 21:09:03 +0200 Subject: [PATCH 01/12] Add paulib export --- .../decidim/admin/exports/_dropdown.html.erb | 9 ++ .../admin/pabulib_exports_controller.rb | 126 ++++++++++++++++ .../budgets/admin/pabulib_export_form.rb | 38 +++++ .../admin/project_bulk_actions_helper.rb | 17 +++ .../admin/pabulib_exports/show.html.erb | 138 ++++++++++++++++++ decidim-budgets/config/locales/en.yml | 77 ++++++++++ .../lib/decidim/budgets/admin_engine.rb | 1 + 7 files changed, 406 insertions(+) create mode 100644 decidim-budgets/app/controllers/decidim/budgets/admin/pabulib_exports_controller.rb create mode 100644 decidim-budgets/app/forms/decidim/budgets/admin/pabulib_export_form.rb create mode 100644 decidim-budgets/app/views/decidim/budgets/admin/pabulib_exports/show.html.erb diff --git a/decidim-admin/app/views/decidim/admin/exports/_dropdown.html.erb b/decidim-admin/app/views/decidim/admin/exports/_dropdown.html.erb index 3bf1f7c0ad1f7..5180fef935d4c 100644 --- a/decidim-admin/app/views/decidim/admin/exports/_dropdown.html.erb +++ b/decidim-admin/app/views/decidim/admin/exports/_dropdown.html.erb @@ -17,5 +17,14 @@ <% end %> <% end %> <% end %> + <% if defined?(extra_export_links) && extra_export_links.is_a?(Array) %> + <% extra_export_links.each do |link| %> + <%= link_to link[:href], method: link[:method] do %> +
  • + <%= t("decidim.admin.exports.export_as", name: t("decidim.#{component.manifest.name}.admin.exports.#{link[:type]}"), export_format: link[:format_name]) %> +
    • + <% end %> + <% end %> + <% end %> diff --git a/decidim-budgets/app/controllers/decidim/budgets/admin/pabulib_exports_controller.rb b/decidim-budgets/app/controllers/decidim/budgets/admin/pabulib_exports_controller.rb new file mode 100644 index 0000000000000..223aa854c9e82 --- /dev/null +++ b/decidim-budgets/app/controllers/decidim/budgets/admin/pabulib_exports_controller.rb @@ -0,0 +1,126 @@ +# frozen_string_literal: true + +module Decidim + module Budgets + module Admin + # This controller allows an admin to export projects from a budget to + # Pabulib format as defined at: https://pabulib.org/format + class PabulibExportsController < Admin::ApplicationController + helper_method :pabulib_vote_type_options, :pabulib_scoring_fn_options + + def show + @form = form(PabulibExportForm).from_params( + description: "#{translated_attribute(current_organization.name)} - #{translated_attribute(current_component.name)} - #{translated_attribute(budget.title)}", + unit: translated_attribute(budget.title), + instance: budget.created_at.strftime("%Y"), + min_length: 1, + max_length: budget.projects.count, + vote_type: "approval" + ) + end + + def create + @form = form(PabulibExportForm).from_params(params) + unless @form.valid? + flash.now[:alert] = I18n.t("pabulib_exports.create.invalid", scope: "decidim.budgets.admin") + return render :new + end + + filename = "decidim-budget-#{budget.id}-results-#{Time.zone.now.strftime("%Y-%m-%d-%H%M%S")}.pb" + response.content_type = "text/plain" + response.headers["Content-Disposition"] = "attachment; filename=#{filename}" + response.headers["Cache-Control"] = "no-cache" + response.headers["Last-Modified"] = Time.now.httpdate + + write("META") + write(key: "value") + write(description: @form.description) + write_attributes(:country, :unit, :instance) + write(num_projects: budget.projects.count) + write(num_votes: budget.orders.finished.count) + write(budget: budget.total_budget) + write(rule: "greedy") # no other rules defined at this point + write(vote_type: @form.vote_type) + + write_attributes(:min_length, :max_length) + write_type_attributes + if budget.orders.any? + write(date_begin: budget.orders.order(:created_at).first.created_at.strftime("%Y-%m-%d")) + write(date_end: budget.orders.order(:created_at).last.created_at.strftime("%Y-%m-%d")) + end + + write("PROJECTS") + write("project_id;name;cost;votes;selected") + budget.projects.each do |project| + votes_amount = Decidim::Budgets::LineItem.joins(:order).where(project:).where.not( + decidim_budgets_orders: { checked_out_at: nil } + ).count + write( + [ + project.id, + translated_attribute(project.title), + project.budget_amount, + votes_amount, + project.selected? ? 1 : 0 + ].join(";") + ) + end + return if budget.orders.none? + + write("VOTES") + write_votes + ensure + response.stream.close + end + + private + + def write(str = nil, **kwargs) + response.write "#{str}\n" if str.present? + return unless kwargs.any? + + response.write "#{kwargs.map { |key, val| [key, val].join(";") }.join(";")}\n" + end + + def write_type_attributes + case @form.vote_type + when "approval" + write_attributes(:min_sum_cost, :max_sum_cost) + when "ordinal" + write_attributes(:scoring_fn) + when "cumulative" + write_attributes(:min_points, :max_points, :min_sum_points, :max_sum_points) + when "scoring" + write_attributes(:min_points, :max_points, :default_score) + end + end + + def write_attributes(*attrs) + attrs.each { |key| write("#{key};#{@form.public_send(key)}") if @form.public_send(key).present? } + end + + # Separated to its own method to allow customization with more specific + # voter data. + def write_votes + write(voter_id: "vote") + budget.orders.finished.each do |order| + # Note that the voter ID is anonymized on purpose according to the + # order ID. The ID of the user could expose their identity e.g. + # through the API. + write([order.id, order.projects.pluck(:id).join(",")].join(";")) + end + end + + def pabulib_vote_type_options + %w(approval ordinal cumulative scoring).map do |type| + [t(type, scope: "activemodel.attributes.pabulib_vote_types", type:), type] + end + end + + def pabulib_scoring_fn_options + ["Borda"] + end + end + end + end +end diff --git a/decidim-budgets/app/forms/decidim/budgets/admin/pabulib_export_form.rb b/decidim-budgets/app/forms/decidim/budgets/admin/pabulib_export_form.rb new file mode 100644 index 0000000000000..fb1f7f796b910 --- /dev/null +++ b/decidim-budgets/app/forms/decidim/budgets/admin/pabulib_export_form.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Decidim + module Budgets + module Admin + # This class holds a Form to create a new pabulib export for projects + # from Decidim's admin panel. + class PabulibExportForm < Decidim::Form + attribute :description, String + attribute :country, String + attribute :unit, String + attribute :instance, String + + attribute :vote_type, String + attribute :min_length, Integer + attribute :max_length, Integer + + # approval + attribute :min_sum_cost, Integer + attribute :max_sum_cost, Integer + + # ordinal + attribute :scoring_fn, String + + # cumulative, scoring + attribute :min_points, Integer + attribute :max_points, Integer + + # cumulative + attribute :min_sum_points, Integer + attribute :max_sum_points, Integer + + # scoring + attribute :default_score, Integer + end + end + end +end diff --git a/decidim-budgets/app/helpers/decidim/budgets/admin/project_bulk_actions_helper.rb b/decidim-budgets/app/helpers/decidim/budgets/admin/project_bulk_actions_helper.rb index be3a0ab564c35..3d8f8b92ca3b8 100644 --- a/decidim-budgets/app/helpers/decidim/budgets/admin/project_bulk_actions_helper.rb +++ b/decidim-budgets/app/helpers/decidim/budgets/admin/project_bulk_actions_helper.rb @@ -14,6 +14,23 @@ def bulk_selections ] ) end + + private + + def render_dropdown(component:, resource_id:, filters:) + render partial: "decidim/admin/exports/dropdown", locals: { component:, resource_id:, filters:, extra_export_links: } + end + + def extra_export_links + [ + { + type: :projects, + format: :pb, + format_name: t("decidim.budgets.admin.exports.formats.pabulib"), + href: budget_pabulib_export_path(budget) + } + ] + end end end end diff --git a/decidim-budgets/app/views/decidim/budgets/admin/pabulib_exports/show.html.erb b/decidim-budgets/app/views/decidim/budgets/admin/pabulib_exports/show.html.erb new file mode 100644 index 0000000000000..3a25f33cf663e --- /dev/null +++ b/decidim-budgets/app/views/decidim/budgets/admin/pabulib_exports/show.html.erb @@ -0,0 +1,138 @@ +<% add_decidim_page_title(t(".title", budget: translated_attribute(budget.title))) %> + +<% format_link = link_to(t(".explanation_link_label_format"), "https://pabulib.org/format", target: "_blank") %> + +
    +

    + <%= t(".title", budget: translated_attribute(budget.title)) %> +

    +
    + +
    +
    + <%= decidim_form_for(@form, url: budget_pabulib_export_path(budget), html: { class: "form form-defaults export_projects" }) do |f| %> +
    +
    +
    +
    + <%= t( + ".explanation_html", + format_link:, + equal_shares_link: link_to(t(".explanation_link_label_equal_shares"), "https://equalshares.net/", target: "_blank"), + tool_link: link_to(t(".explanation_link_label_equal_shares_tool"), "https://equalshares.net/tools/compute/", target: "_blank") + ) %> +
    +
    +
    +
    +
    +
    +

    <%= t(".export_details_general") %>

    +
    + <%= f.text_field :description, label: t(".fields.description.label", field: "description"), help_text: t(".fields.description.help") %> +
    +
    + <%= f.text_field :description, label: t(".fields.country.label", field: "country"), help_text: t(".fields.country.help") %> +
    +
    + <%= f.text_field :unit, label: t(".fields.unit.label", field: "unit"), help_text: t(".fields.unit.help") %> +
    +
    + <%= f.text_field :instance, label: t(".fields.instance.label", field: "instance"), help_text: t(".fields.instance.help") %> +
    +
    +
    +
    +
    +
    +
    +
    +

    <%= t(".export_details_data") %>

    +

    <%= t(".format_info_html", link: format_link) %>

    +
    + +
    + <%= f.select :vote_type, pabulib_vote_type_options, label: t(".fields.vote_type.label", field: "vote_type"), help_text: t(".fields.vote_type.help") %> +
    + +
    + <%= f.number_field :min_length, label: t(".fields.min_length.label", field: "min_length"), help_text: t(".fields.min_length.help", default_value: 1) %> +
    +
    + <%= f.number_field :max_length, label: t(".fields.max_length.label", field: "max_length"), help_text: t(".fields.max_length.help", default_value: "∞") %> +
    + +
    +
    + <%= f.number_field :min_sum_cost, label: t(".fields.min_sum_cost.label", field: "min_sum_cost"), help_text: t(".fields.min_sum_cost.help", default_value: 0) %> +
    +
    + <%= f.number_field :max_sum_cost, label: t(".fields.max_sum_cost.label", field: "max_sum_cost"), help_text: t(".fields.max_sum_cost.help", default_value: "∞") %> +
    +
    + +
    +
    + <%= f.select :scoring_fn, pabulib_scoring_fn_options, label: t(".fields.scoring_fn.label", field: "scoring_fn"), help_text: t(".fields.scoring_fn.help", default_value: pabulib_scoring_fn_options.first) %> +
    +
    + +
    +
    + <%= f.number_field :min_sum_points, label: t(".fields.min_sum_points.label", field: "min_sum_points"), help_text: t(".fields.min_sum_points.help", default_value: 0) %> +
    +
    + <%= f.number_field :max_sum_points, label: t(".fields.max_sum_points.label", field: "max_sum_points"), help_text: t(".fields.max_sum_points.help") %> +
    +
    + +
    +
    + <%= f.number_field :min_points, label: t(".fields.min_points.label", field: "min_points"), help_text: t(".fields.min_points.help", default_value: 0) %> +
    +
    + <%= f.number_field :max_points, label: t(".fields.max_points.label", field: "max_points"), help_text: t(".fields.max_points.help", default_value: "max_sum_points") %> +
    +
    + +
    +
    + <%= f.number_field :default_score, label: t(".fields.default_score.label", field: "default_score"), help_text: t(".fields.default_score.help", default_value: 0) %> +
    +
    +
    +
    +
    +
    + +
    +
    + <%= link_to t(".cancel"), budget_projects_path(budget), class: "button button__sm button__secondary" %> + <%= f.submit t(".create"), class: "button button__sm button__secondary" %> +
    +
    + <% end %> +
    + + + diff --git a/decidim-budgets/config/locales/en.yml b/decidim-budgets/config/locales/en.yml index d2c8be16e432e..e405550c5af77 100644 --- a/decidim-budgets/config/locales/en.yml +++ b/decidim-budgets/config/locales/en.yml @@ -7,6 +7,11 @@ en: title: Title total_budget: Total budget weight: Order position + pabulib_vote_types: + approval: Approval (%{type}) + cumulative: Cumulative (%{type}) + ordinal: Ordinal (%{type}) + scoring: Scoring (%{type}) project: budget_amount: Budget amount description: Description @@ -77,12 +82,84 @@ en: invalid: There was a problem updating this budget. success: Budget successfully updated. exports: + formats: + pabulib: Pabulib projects: Projects models: budget: name: Budget project: name: Project + pabulib_exports: + pabulib_exports: + create: + invalid: Please correct the errors on the form. + show: + cancel: Cancel + create: Export + explanation_html: | +

    + This view will create a %{format_link} export for the participatory budgeting voting results. +

    +

    + This export can be used to create a voting result for the participatory budgeting voting using the %{equal_shares_link}. +

    +

    + In order to calculate the voting result using this method, you can use the %{tool_link}. +

    + explanation_link_label_format: Pabulib format + explanation_link_label_equal_shares: Method of Equal Shares + explanation_link_label_equal_shares_tool: online computation tool + export_details_data: Data attributes + export_details_general: General details + fields: + country: + help: The country where the voting was held. + label: Country (%{field}) + default_score: + help: "The default score for the scoring vote type (default: %{default_value})." + label: Default score (%{field}) + description: + help: Description of the voting describing what was voted on. + label: Description (%{field}) + instance: + help: The instance of the voting, for example, for yearly recurring votings would be the year of voting. + label: Instance (%{field}) + max_length: + help: "Maximum amount of projects to be selected (default: %{default_value})." + label: Maximum length (%{field}) + min_length: + help: "Minimum amount of projects to be selected (default: %{default_value})." + label: Minimum length (%{field}) + max_points: + help: "Maximum points assigned to the selected projects (default: %{default_value})." + label: Maximum points (%{field}) + min_points: + help: "Minimum points assigned to the selected projects (default: %{default_value})." + label: Minimum points (%{field}) + max_sum_cost: + help: "Maximum sum for the cost of a project (default: %{default_value})." + label: Maximum sum cost (%{field}) + min_sum_cost: + help: "Minimum sum for the cost of a project (default: %{default_value})." + label: Minimum sum cost (%{field}) + max_sum_points: + help: Maximum sum of the assigned points for each project. + label: Maximum sum points (%{field}) + min_sum_points: + help: "Minimum sum of the assigned points for each project (default: %{default_value})." + label: Minimum sum points (%{field}) + scoring_fn: + help: "Scoring function for the ordinal type voting (default: %{default_value})." + label: Scoring function (%{field}) + unit: + help: The voting unit within the voting, for example, an area. + label: Unit (%{field}) + vote_type: + help: How are the voting results evaluated (see the documentation for more information). + label: Vote type (%{field}) + format_info_html: Please refer to the %{link} documentation to learn more about these details. + title: Export voting results for %{budget} to Pabulib format projects: create: invalid: There was a problem creating this project. diff --git a/decidim-budgets/lib/decidim/budgets/admin_engine.rb b/decidim-budgets/lib/decidim/budgets/admin_engine.rb index 46130d1c861d6..ca35cc6349e60 100644 --- a/decidim-budgets/lib/decidim/budgets/admin_engine.rb +++ b/decidim-budgets/lib/decidim/budgets/admin_engine.rb @@ -20,6 +20,7 @@ class AdminEngine < ::Rails::Engine post :update_selected post :update_budget resource :proposals_import, only: [:new, :create] + resource :pabulib_export, only: [:show, :create] end end end From 9f4a9d6a40d3c655dad440b100f51af29cc90417 Mon Sep 17 00:00:00 2001 From: Antti Hukkanen Date: Fri, 15 Nov 2024 21:26:11 +0200 Subject: [PATCH 02/12] Fix the country field --- .../views/decidim/budgets/admin/pabulib_exports/show.html.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/decidim-budgets/app/views/decidim/budgets/admin/pabulib_exports/show.html.erb b/decidim-budgets/app/views/decidim/budgets/admin/pabulib_exports/show.html.erb index 3a25f33cf663e..53e15be68786f 100644 --- a/decidim-budgets/app/views/decidim/budgets/admin/pabulib_exports/show.html.erb +++ b/decidim-budgets/app/views/decidim/budgets/admin/pabulib_exports/show.html.erb @@ -32,7 +32,7 @@ <%= f.text_field :description, label: t(".fields.description.label", field: "description"), help_text: t(".fields.description.help") %>
    - <%= f.text_field :description, label: t(".fields.country.label", field: "country"), help_text: t(".fields.country.help") %> + <%= f.text_field :country, label: t(".fields.country.label", field: "country"), help_text: t(".fields.country.help") %>
    <%= f.text_field :unit, label: t(".fields.unit.label", field: "unit"), help_text: t(".fields.unit.help") %> From acdda92d4cf19005f9841a625ea2d1437bcb776f Mon Sep 17 00:00:00 2001 From: Antti Hukkanen Date: Fri, 15 Nov 2024 21:41:24 +0200 Subject: [PATCH 03/12] Allow the `pabulib` term --- .github/actions/spelling/allow.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/spelling/allow.txt b/.github/actions/spelling/allow.txt index 6bb78923e6010..e46452ea22ff4 100644 --- a/.github/actions/spelling/allow.txt +++ b/.github/actions/spelling/allow.txt @@ -4,3 +4,4 @@ https ssh ubuntu workarounds +pabulib From 299dea8662b3fec554f2eed8d88393b86581fbe7 Mon Sep 17 00:00:00 2001 From: Antti Hukkanen Date: Fri, 15 Nov 2024 21:41:50 +0200 Subject: [PATCH 04/12] Normalize locales and fix i18n issues --- decidim-budgets/config/locales/en.yml | 37 +++++++++++++-------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/decidim-budgets/config/locales/en.yml b/decidim-budgets/config/locales/en.yml index e405550c5af77..5023b8f8fe328 100644 --- a/decidim-budgets/config/locales/en.yml +++ b/decidim-budgets/config/locales/en.yml @@ -91,9 +91,8 @@ en: project: name: Project pabulib_exports: - pabulib_exports: - create: - invalid: Please correct the errors on the form. + create: + invalid: Please correct the errors on the form. show: cancel: Cancel create: Export @@ -107,9 +106,9 @@ en:

    In order to calculate the voting result using this method, you can use the %{tool_link}.

    - explanation_link_label_format: Pabulib format explanation_link_label_equal_shares: Method of Equal Shares explanation_link_label_equal_shares_tool: online computation tool + explanation_link_label_format: Pabulib format export_details_data: Data attributes export_details_general: General details fields: @@ -117,7 +116,7 @@ en: help: The country where the voting was held. label: Country (%{field}) default_score: - help: "The default score for the scoring vote type (default: %{default_value})." + help: 'The default score for the scoring vote type (default: %{default_value}).' label: Default score (%{field}) description: help: Description of the voting describing what was voted on. @@ -126,31 +125,31 @@ en: help: The instance of the voting, for example, for yearly recurring votings would be the year of voting. label: Instance (%{field}) max_length: - help: "Maximum amount of projects to be selected (default: %{default_value})." + help: 'Maximum amount of projects to be selected (default: %{default_value}).' label: Maximum length (%{field}) - min_length: - help: "Minimum amount of projects to be selected (default: %{default_value})." - label: Minimum length (%{field}) max_points: - help: "Maximum points assigned to the selected projects (default: %{default_value})." + help: 'Maximum points assigned to the selected projects (default: %{default_value}).' label: Maximum points (%{field}) - min_points: - help: "Minimum points assigned to the selected projects (default: %{default_value})." - label: Minimum points (%{field}) max_sum_cost: - help: "Maximum sum for the cost of a project (default: %{default_value})." + help: 'Maximum sum for the cost of a project (default: %{default_value}).' label: Maximum sum cost (%{field}) - min_sum_cost: - help: "Minimum sum for the cost of a project (default: %{default_value})." - label: Minimum sum cost (%{field}) max_sum_points: help: Maximum sum of the assigned points for each project. label: Maximum sum points (%{field}) + min_length: + help: 'Minimum amount of projects to be selected (default: %{default_value}).' + label: Minimum length (%{field}) + min_points: + help: 'Minimum points assigned to the selected projects (default: %{default_value}).' + label: Minimum points (%{field}) + min_sum_cost: + help: 'Minimum sum for the cost of a project (default: %{default_value}).' + label: Minimum sum cost (%{field}) min_sum_points: - help: "Minimum sum of the assigned points for each project (default: %{default_value})." + help: 'Minimum sum of the assigned points for each project (default: %{default_value}).' label: Minimum sum points (%{field}) scoring_fn: - help: "Scoring function for the ordinal type voting (default: %{default_value})." + help: 'Scoring function for the ordinal type voting (default: %{default_value}).' label: Scoring function (%{field}) unit: help: The voting unit within the voting, for example, an area. From f8fef62a2135fa1514321846c878fbb469be39cc Mon Sep 17 00:00:00 2001 From: Antti Hukkanen Date: Sat, 16 Nov 2024 18:41:21 +0200 Subject: [PATCH 05/12] Make the controller thinner Move the pabulib export logic to its own module. --- .../admin/pabulib_exports_controller.rb | 83 ++--------------- decidim-budgets/lib/decidim/budgets.rb | 1 + .../lib/decidim/budgets/pabulib.rb | 37 ++++++++ .../lib/decidim/budgets/pabulib/exporter.rb | 79 ++++++++++++++++ .../lib/decidim/budgets/pabulib/writer.rb | 93 +++++++++++++++++++ 5 files changed, 216 insertions(+), 77 deletions(-) create mode 100644 decidim-budgets/lib/decidim/budgets/pabulib.rb create mode 100644 decidim-budgets/lib/decidim/budgets/pabulib/exporter.rb create mode 100644 decidim-budgets/lib/decidim/budgets/pabulib/writer.rb diff --git a/decidim-budgets/app/controllers/decidim/budgets/admin/pabulib_exports_controller.rb b/decidim-budgets/app/controllers/decidim/budgets/admin/pabulib_exports_controller.rb index 223aa854c9e82..a88c85d8d1588 100644 --- a/decidim-budgets/app/controllers/decidim/budgets/admin/pabulib_exports_controller.rb +++ b/decidim-budgets/app/controllers/decidim/budgets/admin/pabulib_exports_controller.rb @@ -27,98 +27,27 @@ def create end filename = "decidim-budget-#{budget.id}-results-#{Time.zone.now.strftime("%Y-%m-%d-%H%M%S")}.pb" - response.content_type = "text/plain" - response.headers["Content-Disposition"] = "attachment; filename=#{filename}" + response.content_type = "text/csv" + response.headers["Content-Disposition"] = %(attachment; filename="#{filename}") response.headers["Cache-Control"] = "no-cache" response.headers["Last-Modified"] = Time.now.httpdate - write("META") - write(key: "value") - write(description: @form.description) - write_attributes(:country, :unit, :instance) - write(num_projects: budget.projects.count) - write(num_votes: budget.orders.finished.count) - write(budget: budget.total_budget) - write(rule: "greedy") # no other rules defined at this point - write(vote_type: @form.vote_type) - - write_attributes(:min_length, :max_length) - write_type_attributes - if budget.orders.any? - write(date_begin: budget.orders.order(:created_at).first.created_at.strftime("%Y-%m-%d")) - write(date_end: budget.orders.order(:created_at).last.created_at.strftime("%Y-%m-%d")) - end - - write("PROJECTS") - write("project_id;name;cost;votes;selected") - budget.projects.each do |project| - votes_amount = Decidim::Budgets::LineItem.joins(:order).where(project:).where.not( - decidim_budgets_orders: { checked_out_at: nil } - ).count - write( - [ - project.id, - translated_attribute(project.title), - project.budget_amount, - votes_amount, - project.selected? ? 1 : 0 - ].join(";") - ) - end - return if budget.orders.none? - - write("VOTES") - write_votes + exporter = Pabulib::Exporter.new(@form) + exporter.export(budget, response) ensure response.stream.close end private - def write(str = nil, **kwargs) - response.write "#{str}\n" if str.present? - return unless kwargs.any? - - response.write "#{kwargs.map { |key, val| [key, val].join(";") }.join(";")}\n" - end - - def write_type_attributes - case @form.vote_type - when "approval" - write_attributes(:min_sum_cost, :max_sum_cost) - when "ordinal" - write_attributes(:scoring_fn) - when "cumulative" - write_attributes(:min_points, :max_points, :min_sum_points, :max_sum_points) - when "scoring" - write_attributes(:min_points, :max_points, :default_score) - end - end - - def write_attributes(*attrs) - attrs.each { |key| write("#{key};#{@form.public_send(key)}") if @form.public_send(key).present? } - end - - # Separated to its own method to allow customization with more specific - # voter data. - def write_votes - write(voter_id: "vote") - budget.orders.finished.each do |order| - # Note that the voter ID is anonymized on purpose according to the - # order ID. The ID of the user could expose their identity e.g. - # through the API. - write([order.id, order.projects.pluck(:id).join(",")].join(";")) - end - end - def pabulib_vote_type_options - %w(approval ordinal cumulative scoring).map do |type| + Pabulib::VOTE_TYPES.map do |type| [t(type, scope: "activemodel.attributes.pabulib_vote_types", type:), type] end end def pabulib_scoring_fn_options - ["Borda"] + Pabulib::SCORING_FNS end end end diff --git a/decidim-budgets/lib/decidim/budgets.rb b/decidim-budgets/lib/decidim/budgets.rb index 20a9d5b88659e..dae7b041da045 100644 --- a/decidim-budgets/lib/decidim/budgets.rb +++ b/decidim-budgets/lib/decidim/budgets.rb @@ -11,6 +11,7 @@ module Decidim # Base module for this engine. module Budgets autoload :ProjectSerializer, "decidim/budgets/project_serializer" + autoload :Pabulib, "decidim/budgets/pabulib" include ActiveSupport::Configurable diff --git a/decidim-budgets/lib/decidim/budgets/pabulib.rb b/decidim-budgets/lib/decidim/budgets/pabulib.rb new file mode 100644 index 0000000000000..be315594d8a43 --- /dev/null +++ b/decidim-budgets/lib/decidim/budgets/pabulib.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Decidim + module Budgets + module Pabulib + autoload :Exporter, "decidim/budgets/pabulib/exporter" + autoload :Writer, "decidim/budgets/pabulib/writer" + + VOTE_TYPES = %w(approval ordinal cumulative scoring).freeze + SCORING_FNS = %w(Borda).freeze + Metadata = Struct.new( + :description, + :country, + :unit, + :instance, + :num_projects, + :num_votes, + :budget, + :vote_type, + :min_length, + :max_length, + :min_sum_cost, + :max_sum_cost, + :scoring_fn, + :min_points, + :max_points, + :min_sum_points, + :max_sum_points, + :default_score, + :date_begin, + :date_end + ) + Project = Struct.new(:project_id, :name, :cost, :votes, :selected) + Vote = Struct.new(:voter_id, :vote) + end + end +end diff --git a/decidim-budgets/lib/decidim/budgets/pabulib/exporter.rb b/decidim-budgets/lib/decidim/budgets/pabulib/exporter.rb new file mode 100644 index 0000000000000..9f91b22889008 --- /dev/null +++ b/decidim-budgets/lib/decidim/budgets/pabulib/exporter.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +module Decidim + module Budgets + module Pabulib + # Exports a single budget to the Paulib format (.pb). + class Exporter + include Decidim::TranslatableAttributes + + def initialize(config) + @config = config + end + + def export(budget, io) + writer = Pabulib::Writer.new(io, create_metadata_for(budget)) + writer.write_metadata + writer.write_projects(budget.projects) { |project| convert_project(project) } + writer.write_votes(budget.orders.finished) { |order| convert_vote(order) } + end + + private + + attr_reader :config + + def create_metadata_for(budget) + Pabulib::Metadata.new( + description: config.description, + country: config.country, + unit: config.unit, + instance: config.instance, + num_projects: budget.projects.count, + num_votes: budget.orders.finished.count, + budget: budget.total_budget, + vote_type: config.vote_type, + min_length: config.min_length.presence || 1, + max_length: config.max_length.presence || budget.projects.count, + min_sum_cost: config.min_sum_cost, + max_sum_cost: config.max_sum_cost, + scoring_fn: config.scoring_fn, + min_points: config.min_points, + max_points: config.max_points, + min_sum_points: config.min_sum_points, + max_sum_points: config.max_sum_points, + default_score: config.default_score + ).tap do |metadata| + if budget.orders.any? + metadata.date_begin = budget.orders.order(:created_at).first.created_at + metadata.date_end = budget.orders.order(:created_at).last.created_at + end + end + end + + def convert_project(project) + votes_amount = Decidim::Budgets::LineItem.joins(:order).where(project:).where.not( + decidim_budgets_orders: { checked_out_at: nil } + ).count + + Pabulib::Project.new( + project_id: project.id, + name: translated_attribute(project.title), + cost: project.budget_amount, + votes: votes_amount, + selected: project.selected? ? 1 : 0 + ) + end + + def convert_vote(order) + # Note that the voter ID is anonymized on purpose according to the + # order ID. The ID of the user could expose their identity e.g. + # through the API. + Pabulib::Vote.new( + voter_id: order.id, + vote: order.projects.pluck(:id).join(",") + ) + end + end + end + end +end diff --git a/decidim-budgets/lib/decidim/budgets/pabulib/writer.rb b/decidim-budgets/lib/decidim/budgets/pabulib/writer.rb new file mode 100644 index 0000000000000..10909684140f2 --- /dev/null +++ b/decidim-budgets/lib/decidim/budgets/pabulib/writer.rb @@ -0,0 +1,93 @@ +# frozen_string_literal: true + +module Decidim + module Budgets + module Pabulib + # Creates the PB voting export in Pabulib format (.pb) for a participatory + # budgeting budget. Note that the Pabulib format currently supports only a + # single budget at a time which is why this only exports a single budget + # at a time. + class Writer + def initialize(io, metadata) + @io = io + @metadata = metadata + end + + def write_metadata + raise InvalidMetadataError, "Description not defined." if metadata.description.blank? + + write("META") + write(key: "value") + write(description: metadata.description) + write_attributes(metadata, :country, :unit, :instance) + write(num_projects: metadata.num_projects) + write(num_votes: metadata.num_votes) + write(budget: metadata.budget) + write(rule: "greedy") # no other rules defined at this point + write(vote_type: metadata.vote_type) + + write_attributes(metadata, :min_length, :max_length) + write_type_attributes + + if metadata.date_begin && metadata.date_end + write(date_begin: metadata.date_begin.strftime("%Y-%m-%d")) + write(date_end: metadata.date_end.strftime("%Y-%m-%d")) + end + end + + def write_projects(data, &) + write_data("PROJECTS", data, &) + end + + def write_votes(data, &) + write_data("VOTES", data, &) + end + + private + + attr_reader :io, :metadata + + def write(str = nil, **kwargs) + io.write "#{str}\n" if str.present? + return unless kwargs.any? + + io.write "#{kwargs.map { |key, val| [key, val].join(";") }.join(";")}\n" + end + + def write_type_attributes + case metadata.vote_type + when "approval" + write_attributes(metadata, :min_sum_cost, :max_sum_cost) + when "ordinal" + write_attributes(metadata, :scoring_fn) + when "cumulative" + write_attributes(metadata, :min_points, :max_points, :min_sum_points, :max_sum_points) + when "scoring" + write_attributes(metadata, :min_points, :max_points, :default_score) + else + raise InvalidMetadataError, "Unknown vote_type: #{metadata.vote_type}" + end + end + + def write_attributes(source, *attrs) + attrs.each { |key| write("#{key};#{source.public_send(key)}") if source.public_send(key).present? } + end + + def write_data(section, data) + return if data.empty? + + write(section) + data.each_with_index do |item, idx| + struct = yield item + write(struct.members.join(";")) if idx.zero? + write(struct.members.map { |key| struct.public_send(key) }.join(";")) + end + end + + class Error < StandardError; end + + class InvalidMetadataError < Error; end + end + end + end +end From dcc73774d040a406c77adb18e2775e882289d17f Mon Sep 17 00:00:00 2001 From: Antti Hukkanen Date: Sat, 16 Nov 2024 18:51:11 +0200 Subject: [PATCH 06/12] Fix double margin on the export page --- .../admin/pabulib_exports/show.html.erb | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/decidim-budgets/app/views/decidim/budgets/admin/pabulib_exports/show.html.erb b/decidim-budgets/app/views/decidim/budgets/admin/pabulib_exports/show.html.erb index 53e15be68786f..c2da052ecdf33 100644 --- a/decidim-budgets/app/views/decidim/budgets/admin/pabulib_exports/show.html.erb +++ b/decidim-budgets/app/views/decidim/budgets/admin/pabulib_exports/show.html.erb @@ -28,18 +28,19 @@

    <%= t(".export_details_general") %>

    -
    - <%= f.text_field :description, label: t(".fields.description.label", field: "description"), help_text: t(".fields.description.help") %> -
    -
    - <%= f.text_field :country, label: t(".fields.country.label", field: "country"), help_text: t(".fields.country.help") %> -
    -
    - <%= f.text_field :unit, label: t(".fields.unit.label", field: "unit"), help_text: t(".fields.unit.help") %> -
    -
    - <%= f.text_field :instance, label: t(".fields.instance.label", field: "instance"), help_text: t(".fields.instance.help") %> -
    +
    + +
    + <%= f.text_field :description, label: t(".fields.description.label", field: "description"), help_text: t(".fields.description.help") %> +
    +
    + <%= f.text_field :country, label: t(".fields.country.label", field: "country"), help_text: t(".fields.country.help") %> +
    +
    + <%= f.text_field :unit, label: t(".fields.unit.label", field: "unit"), help_text: t(".fields.unit.help") %> +
    +
    + <%= f.text_field :instance, label: t(".fields.instance.label", field: "instance"), help_text: t(".fields.instance.help") %>
    From 98bfae920b91bbc093388dc5e25fe2df450f941b Mon Sep 17 00:00:00 2001 From: Antti Hukkanen Date: Sat, 16 Nov 2024 18:51:39 +0200 Subject: [PATCH 07/12] Name the export type differently (as it's not only projects) --- .../decidim/budgets/admin/project_bulk_actions_helper.rb | 2 +- decidim-budgets/config/locales/en.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/decidim-budgets/app/helpers/decidim/budgets/admin/project_bulk_actions_helper.rb b/decidim-budgets/app/helpers/decidim/budgets/admin/project_bulk_actions_helper.rb index 3d8f8b92ca3b8..fdb8b5126ea2a 100644 --- a/decidim-budgets/app/helpers/decidim/budgets/admin/project_bulk_actions_helper.rb +++ b/decidim-budgets/app/helpers/decidim/budgets/admin/project_bulk_actions_helper.rb @@ -24,7 +24,7 @@ def render_dropdown(component:, resource_id:, filters:) def extra_export_links [ { - type: :projects, + type: :voting_results, format: :pb, format_name: t("decidim.budgets.admin.exports.formats.pabulib"), href: budget_pabulib_export_path(budget) diff --git a/decidim-budgets/config/locales/en.yml b/decidim-budgets/config/locales/en.yml index 5023b8f8fe328..bae3ddfec68d5 100644 --- a/decidim-budgets/config/locales/en.yml +++ b/decidim-budgets/config/locales/en.yml @@ -85,6 +85,7 @@ en: formats: pabulib: Pabulib projects: Projects + voting_results: Voting results models: budget: name: Budget From 1066bf684ea7ce16842bb1b1075dd6cddf4ecf36 Mon Sep 17 00:00:00 2001 From: Antti Hukkanen Date: Sat, 16 Nov 2024 19:51:27 +0200 Subject: [PATCH 08/12] Add rake task for the pabulib export --- .../lib/tasks/decidim_budgets_tasks.rake | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 decidim-budgets/lib/tasks/decidim_budgets_tasks.rake diff --git a/decidim-budgets/lib/tasks/decidim_budgets_tasks.rake b/decidim-budgets/lib/tasks/decidim_budgets_tasks.rake new file mode 100644 index 0000000000000..c0f21085080ca --- /dev/null +++ b/decidim-budgets/lib/tasks/decidim_budgets_tasks.rake @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +namespace :decidim do + namespace :budgets do + namespace :export do + desc "Export voting results to Pabulib format" + task :budget_pabulib, [:budget_id, :output_path] => :environment do |_, args| + if args.budget_id.blank? + puts "Please define the budget ID to export as the first argument." + next + end + if args.output_path.blank? + puts "Please define the output path as the second argument (e.g. tmp/budget-results-#{args.budget_id}.pb)." + next + end + if File.exist?(args.output_path) + print "File already exists at the defined output path. Do you want to override it? [y/N] " + answer = $stdin.gets.strip + unless %w(y Y yes).include?(answer) + puts "Export cancelled." + next + end + end + + budget = Decidim::Budgets::Budget.find_by(id: args[:budget_id]) + unless budget + puts "Invalid budget ID: #{args[:budget_id]}" + next + end + + organization = budget.organization + preferred_locale = ENV.fetch("DECIDIM_LOCALE", "") + translated_attribute = ->(value) { value[preferred_locale] || value[organization.default_locale] || value.values.first } + + component = budget.component + config = { + description: "#{translated_attribute.call(organization.name)} - #{translated_attribute.call(component.name)} - #{translated_attribute.call(budget.title)}", + unit: translated_attribute.call(budget.title), + instance: budget.created_at.strftime("%Y"), + min_length: 1, + max_length: budget.projects.count, + vote_type: "approval" + } + args.extras.each do |configdef| + key, value = configdef.split("=") + config[key.to_sym] = value + end + + form = Decidim::Budgets::Admin::PabulibExportForm.from_params(config) + exporter = Decidim::Budgets::Pabulib::Exporter.new(form) + File.open(args.output_path, "w") { |file| exporter.export(budget, file) } + + puts %(Exported budget "#{translated_attribute.call(budget.title)}" (ID: #{budget.id}) to: #{args.output_path}) + end + end + end +end From 4a7bdfca14e28ca27a2959f9df657428ab90b41f Mon Sep 17 00:00:00 2001 From: Antti Hukkanen Date: Sat, 16 Nov 2024 19:51:41 +0200 Subject: [PATCH 09/12] Add warning for large datasets to use the rake task --- .../budgets/admin/pabulib_exports/show.html.erb | 12 ++++++++++++ decidim-budgets/config/locales/en.yml | 3 +++ 2 files changed, 15 insertions(+) diff --git a/decidim-budgets/app/views/decidim/budgets/admin/pabulib_exports/show.html.erb b/decidim-budgets/app/views/decidim/budgets/admin/pabulib_exports/show.html.erb index c2da052ecdf33..7a4ea25f0f230 100644 --- a/decidim-budgets/app/views/decidim/budgets/admin/pabulib_exports/show.html.erb +++ b/decidim-budgets/app/views/decidim/budgets/admin/pabulib_exports/show.html.erb @@ -8,6 +8,18 @@ +<% if budget.orders.finished.count > 10000 %> + <%= cell( + "decidim/announcement", + t( + ".large_dataset_warning_html", + amount: budget.orders.finished.count, + command: "rake decidim:budgets:export:budget_pabulib[#{budget.id},tmp/budget-results-#{budget.id}.pb]" + ), + callout_class: "alert" + ) %> +<% end %> +
    <%= decidim_form_for(@form, url: budget_pabulib_export_path(budget), html: { class: "form form-defaults export_projects" }) do |f| %> diff --git a/decidim-budgets/config/locales/en.yml b/decidim-budgets/config/locales/en.yml index bae3ddfec68d5..2263ab1de6f2d 100644 --- a/decidim-budgets/config/locales/en.yml +++ b/decidim-budgets/config/locales/en.yml @@ -159,6 +159,9 @@ en: help: How are the voting results evaluated (see the documentation for more information). label: Vote type (%{field}) format_info_html: Please refer to the %{link} documentation to learn more about these details. + large_dataset_warning_html: | +

    Your dataset contains %{amount} votes and is therefore very large for exporting through this user interface.

    +

    Please consider exporting your dataset from the console with the following command: %{command}

    title: Export voting results for %{budget} to Pabulib format projects: create: From d343c433d3ae5b8b6389568cea0ef24fc830b99e Mon Sep 17 00:00:00 2001 From: Antti Hukkanen Date: Sat, 16 Nov 2024 19:58:55 +0200 Subject: [PATCH 10/12] If locale is not defined, fetch it from the system ENV vars --- decidim-budgets/lib/tasks/decidim_budgets_tasks.rake | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/decidim-budgets/lib/tasks/decidim_budgets_tasks.rake b/decidim-budgets/lib/tasks/decidim_budgets_tasks.rake index c0f21085080ca..da323f895ca9c 100644 --- a/decidim-budgets/lib/tasks/decidim_budgets_tasks.rake +++ b/decidim-budgets/lib/tasks/decidim_budgets_tasks.rake @@ -29,8 +29,11 @@ namespace :decidim do end organization = budget.organization - preferred_locale = ENV.fetch("DECIDIM_LOCALE", "") - translated_attribute = ->(value) { value[preferred_locale] || value[organization.default_locale] || value.values.first } + preferred_locale = ENV.fetch("DECIDIM_LOCALE", ENV.fetch("LANGUAGE", ENV.fetch("LANG", ""))).split(".").first.sub("_", "-") + preferred_locale_short = preferred_locale.split("-").first + translated_attribute = lambda do |value| + value[preferred_locale] || value[preferred_locale_short] || value[organization.default_locale] || value.values.first + end component = budget.component config = { From a90c0d24eaaeace8b5642c1e1ddde409885d1987 Mon Sep 17 00:00:00 2001 From: Antti Hukkanen Date: Sat, 16 Nov 2024 20:03:10 +0200 Subject: [PATCH 11/12] Add usage documentation for the rake task --- decidim-budgets/lib/tasks/decidim_budgets_tasks.rake | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/decidim-budgets/lib/tasks/decidim_budgets_tasks.rake b/decidim-budgets/lib/tasks/decidim_budgets_tasks.rake index da323f895ca9c..ad55f8fed6f69 100644 --- a/decidim-budgets/lib/tasks/decidim_budgets_tasks.rake +++ b/decidim-budgets/lib/tasks/decidim_budgets_tasks.rake @@ -3,6 +3,15 @@ namespace :decidim do namespace :budgets do namespace :export do + # Usage: + # rake decidim:budgets:export:budget_pabulib[123,tmp/budget-results-123.pb] + # + # As extra arguments you can define the configuration options for the + # export with the `key=value` format, for example: + # rake decidim:budgets:export:budget_pabulib[123,tmp/budget-results-123.pb,description="My export",unit="South District"] + # + # For all available export options, please refer to the arguments + # available at `Decidim::Budgets::Admin::PabulibExportForm`. desc "Export voting results to Pabulib format" task :budget_pabulib, [:budget_id, :output_path] => :environment do |_, args| if args.budget_id.blank? From 55d6dae3793a7c5b999c8f4adb1dfbcfcfb79be9 Mon Sep 17 00:00:00 2001 From: Antti Hukkanen Date: Wed, 20 Nov 2024 10:04:01 +0200 Subject: [PATCH 12/12] Refer to the rake task arguments the same way always --- decidim-budgets/lib/tasks/decidim_budgets_tasks.rake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/decidim-budgets/lib/tasks/decidim_budgets_tasks.rake b/decidim-budgets/lib/tasks/decidim_budgets_tasks.rake index ad55f8fed6f69..f2bd1c6967c7c 100644 --- a/decidim-budgets/lib/tasks/decidim_budgets_tasks.rake +++ b/decidim-budgets/lib/tasks/decidim_budgets_tasks.rake @@ -31,9 +31,9 @@ namespace :decidim do end end - budget = Decidim::Budgets::Budget.find_by(id: args[:budget_id]) + budget = Decidim::Budgets::Budget.find_by(id: args.budget_id) unless budget - puts "Invalid budget ID: #{args[:budget_id]}" + puts "Invalid budget ID: #{args.budget_id}" next end