Skip to content

Add XenForo direct messages importer with improved recipient handling… #31508

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Add XenForo direct messages importer with improved recipient handling… #31508

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
137 changes: 94 additions & 43 deletions script/import_scripts/xenforo.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -308,55 +308,106 @@ def import_posts
end
end

def import_private_messages
puts "", "importing private messages..."
post_count = mysql_query("SELECT COUNT(*) count FROM xf_conversation_message").first["count"]
def import_private_messages
puts "", "Importing private messages..."
total_count = mysql_query("SELECT COUNT(*) AS count FROM xf_conversation_message").first["count"]

batches(BATCH_SIZE) do |offset|
posts = mysql_query <<-SQL
SELECT c.conversation_id, c.recipients, c.title, m.message, m.user_id, m.message_date, m.message_id, IF(c.first_message_id != m.message_id, c.first_message_id, 0) as topic_id
FROM xf_conversation_master c
LEFT JOIN xf_conversation_message m ON m.conversation_id = c.conversation_id
ORDER BY c.conversation_id, m.message_id
LIMIT #{BATCH_SIZE}
OFFSET #{offset}
rows = mysql_query(<<-SQL).to_a
SELECT
cm.conversation_id,
cm.title,
cm.user_id AS conversation_owner,
cm.recipients,
cm.first_message_id,
cmsg.message_id,
cmsg.message,
cmsg.user_id AS message_user_id,
cmsg.message_date,
cmsg.username AS message_username
FROM xf_conversation_master cm
JOIN xf_conversation_message cmsg
ON cmsg.conversation_id = cm.conversation_id
ORDER BY cm.conversation_id, cmsg.message_date
LIMIT #{BATCH_SIZE} OFFSET #{offset}
SQL
break if posts.size < 1
next if all_records_exist? :posts, posts.map { |post| "pm_#{post["message_id"]}" }
create_posts(posts, total: post_count, offset: offset) do |post|
user_id = user_id_from_imported_user_id(post["user_id"]) || Discourse::SYSTEM_USER_ID
title = post["title"]
message_id = "pm_#{post["message_id"]}"
raw = process_xenforo_post(post["message"], 0)
if raw.present?
msg = {
id: message_id,
user_id: user_id,
raw: raw,
created_at: Time.zone.at(post["message_date"].to_i),
import_mode: true,
}
if post["topic_id"] <= 0
topic_id = post["topic_id"]
if t = topic_lookup_from_imported_post_id("pm_#{topic_id}")
msg[:topic_id] = t[:topic_id]
else
puts "Topic ID #{topic_id} not found, skipping post #{post["message_id"]} from #{post["user_id"]}"
next

break if rows.empty?
next if all_records_exist?(:posts, rows.map { |row| "pm_#{row["message_id"]}" })

create_posts(rows, total: total_count, offset: offset) do |row|
# Determine sender: use conversation_owner for the first message; otherwise, use message_user_id.
mapped_user_id = if row["message_id"].to_i == row["first_message_id"].to_i
user_id_from_imported_user_id(row["conversation_owner"])
else
user_id_from_imported_user_id(row["message_user_id"])
end

# If mapping fails, fall back to the system user.
unless User.exists?(id: mapped_user_id)
puts "Row #{row["message_id"]}: User mapping invalid for XenForo user #{row["message_user_id"]} or conversation owner #{row["conversation_owner"]}, using system user."
mapped_user_id = Discourse::SYSTEM_USER_ID
end

# Process the message body and timestamp.
raw = process_xenforo_post(row["message"], row["message_id"])
created_at = Time.zone.at(row["message_date"].to_i)

if row["message_id"].to_i == row["first_message_id"].to_i
# For the first message (PM topic), get target recipients.
recipients_raw = (PHP.unserialize(row["recipients"]) rescue nil)
if recipients_raw.blank?
begin
recipients_raw = JSON.parse(row["recipients"])
rescue
recipients_raw = {}
end
else
msg[:title] = post["title"]
msg[:archetype] = Archetype.private_message
to_user_array = PHP.unserialize(post["recipients"])
if to_user_array.size > 0
discourse_user_ids = to_user_array.keys.map { |id| user_id_from_imported_user_id(id) }
usernames = User.where(id: [discourse_user_ids]).pluck(:username)
msg[:target_usernames] = usernames.join(",")
end

# Fallback: if still empty, query the recipient table.
if recipients_raw.blank? || recipients_raw.empty?
recipient_rows = mysql_query("SELECT user_id FROM #{TABLE_PREFIX}conversation_recipient WHERE conversation_id = #{row["conversation_id"]} AND recipient_state = 'active'").to_a
if recipient_rows.any?
recipients_raw = {}
recipient_rows.each { |r| recipients_raw[r["user_id"].to_s] = true }
end
end
msg

target_usernames = if recipients_raw.present? && recipients_raw.keys.any?
discourse_user_ids = recipients_raw.keys.map { |id| user_id_from_imported_user_id(id.to_s) }
User.where(id: discourse_user_ids).pluck(:username).join(",")
end

if target_usernames.blank?
puts "Row #{row["message_id"]}: No valid target usernames found, skipping conversation."
next
end

{
id: "pm_#{row["message_id"]}",
user_id: mapped_user_id,
title: row["title"].presence || "No title",
raw: raw,
created_at: created_at,
archetype: Archetype.private_message,
import_mode: true,
target_usernames: target_usernames
}
else
puts "Empty message, skipping post #{post["message_id"]}"
next
# For replies, find the topic by the first message.
parent = topic_lookup_from_imported_post_id("pm_#{row["first_message_id"]}")
unless parent
puts "Row #{row["message_id"]}: Topic for conversation #{row["conversation_id"]} not found, skipping message."
next
end
{
id: "pm_#{row["message_id"]}",
user_id: mapped_user_id,
raw: raw,
created_at: created_at,
topic_id: parent[:topic_id],
import_mode: true
}
end
end
end
Expand Down
Loading