Migrating Users and Orgs from Clerk

Migrating your users and organizations from one auth source to another can seem scary. Fortunately, we're here to help! See our Migrating Users to PropelAuth guide for more general information or stay here for an advanced guide, including code snippets, on how to migrate your users and orgs from Clerk to PropelAuth.

Exporting Users from Clerk

The first step of exporting users from Clerk is to request a password export by contacting Clerk support. When complete, you'll be able to download a .txt file from your Clerk dashboard.

Update the export from a .txt to a .csv by updating the file extension from export_example.txt to export_example.csv

While this export does contain your users' password hashes, it could be missing important user data such as their profile picture or any custom user properties. Because of this, we'll create a new export using Clerk's Backend API to get all the user data and then add the password hash export to it. This means we need to install the Requests and Pandas libraries.

pip install requests pandas

Get your Clerk API Secret key (found under "API Keys" in Clerk) and run this script to create a new file called user_data.json containing the rest of your user data

import json
import requests
import pandas as pd

# Replace with your Clerk Secret key
CLERK_API_KEY="b1988511.."

# Edit this field to point to the csv file from Clerk
YOUR_CSV_FILE = './clerk_export.csv'

df = pd.read_csv(YOUR_CSV_FILE)

def get_password_hash(user_id):
    matching_row = df[df['id'] == user_id]
    if not matching_row.empty:
        password_hash = matching_row['password_digest'].iloc[0]
        if pd.isnull(password_hash):
            return None
        return password_hash
    return None

def get_users():
    total_users = []
    more_users = True
    headers = {
        'Authorization': f'Bearer {CLERK_API_KEY}'
    }
    offset=0
    while more_users:
        request = requests.get(url=f"https://api.clerk.com/v1/users?limit=500&offset={offset}", headers=headers)
        users = request.json()
        if len(users) > 0:
            total_users += users
            offset += 500
        else:
            more_users = False
    for user in total_users:
        user['password_hash'] = get_password_hash(user['id'])
        
    return total_users

def export_users():
    all_users = get_users()
    with open("user_data.json", "w") as outfile:
        json.dump(all_users, outfile)
        
if __name__ == "__main__":
    export_users()

Importing Users into PropelAuth

We can then use user_data.json to import your user data to PropelAuth. PropelAuth offers an API endpoint to make migrating users as simple as possible. Before we run the script, let's install PropelAuth's Python library:

pip install propelauth_py

This script will attempt to import each of the users found in your user_data.json file into PropelAuth. If any of the imports fail, it will automatically generate a ./unmigrated_users.json file for you.

To run the script we need to get your PROPELAUTH_AUTH_URL and PROPELAUTH_AUTH_API_KEY, both of which can be found in your Backend Integration page in PropelAuth.

Some of your users might not have passwords because they use OAuth/Social logins. If that’s the case, they can still log in via your social logins or magic links.

from propelauth_py import init_base_auth
import json

# Replace with your PropelAuth Auth URL and API key
PROPELAUTH_AUTH_URL = "https://auth.yourdomain.com"
PROPELAUTH_AUTH_API_KEY = "b1988511..."

# replace with your JSON path
user_json_filename = "./user_data.json"

# If we fail to migrate any users, we'll output them here
unmigrated_users_filename = "./unmigrated_users.json"

auth = init_base_auth(PROPELAUTH_AUTH_URL, PROPELAUTH_AUTH_API_KEY)
    
def create_propelauth_user(user_data):
    auth.migrate_user_from_external_source(
        email = user_data.get('email'),
        email_confirmed = user_data.get('email_confirmed'),
        existing_user_id = user_data.get('id'),
        existing_password_hash = user_data.get('password_hash'),
        ask_user_to_update_password_on_login = False,
        enabled = True,
        username = user_data.get('username'),
        first_name = user_data.get('first_name'),
        last_name = user_data.get('last_name'),
        properties = user_data.get('properties'),
    )
    print(f"API call successful for user: {user_data.get('id')}")

def get_primary_email(email_id, emails):
    for email in emails:
        if email.get('id') == email_id:
            is_confirmed = False
            if email.get('verification').get('status') == 'verified':
                is_confirmed = True
            return { 
                "email": email.get('email_address'),
                "is_confirmed": is_confirmed
            }

def process_users():
    users_with_errors = []
    
    with open(user_json_filename, "r") as user_json_file:
        user_data = json.load(user_json_file)
        # Iterate through each user row
        for user in user_data:
            email_dict = get_primary_email(user.get('primary_email_address_id'), user.get('email_addresses'))
            try:
                user_migrate_data = {
                    'id': user.get('id'),
                    'email': email_dict.get('email'),
					'email_confirmed': email_dict.get('is_confirmed'),
                    'password_hash': user.get('password_hash'),
                    
                    # Fields below this are dependent on your setup, you may need to modify
                    #'username': row.get('username'),
                    #'first_name': row.get('first_name'),
                    #'last_name': row.get('last_name'),
                    # Any custom user properties that aren't first_name, last_name, or username go here
                    #'properties': {
                    #    'sample_custom_property': row['sample']
                    #}
                }                
                create_propelauth_user(user_migrate_data)
            except Exception as e:
                users_with_errors.append(user)
                print(f"Error during processing user: {user['Id']}")
                print(f"Error: {e}")

    if len(users_with_errors) > 0:
        with open(unmigrated_users_filename, "w") as unmigrated_users_file:
            for failed_user in users_with_errors:
                unmigrated_users_file.write(json.dumps(failed_user) + "\n")

        print(f"We couldn't migrate all users - the users we failed to migrate are here {unmigrated_users_filename}")

# Run the script
if __name__ == "__main__":
    process_users()

Need a do over? Here's a script to quickly delete all of your users so you can try again.

import requests
from propelauth_py import init_base_auth

# Replace with your PropelAuth Auth URL and API key
PROPELAUTH_AUTH_URL = "https://auth.yourdomain.com"
PROPELAUTH_AUTH_API_KEY = "b1988511..."

auth = init_base_auth(PROPELAUTH_AUTH_URL, PROPELAUTH_AUTH_API_KEY)

def get_users():
    try:
        response = auth.fetch_users_by_query(
            page_size = 10,
            page_number = 0
        )
        return response
    except requests.exceptions.RequestException as e:
        print(f"Error making API call to retrieve users")
        print(f"Error: {e}")

def delete_user(user_id):
    try:
        auth.delete_user(user_id)
    except requests.exceptions.RequestException as e:
        print(f"Error making API call to delete user {user_id}")
        print(f"Error: {e}")

def delete_users():
    user_response = get_users()
    while len(user_response['users']) > 0:
        for user in user_response['users']:
            delete_user(user['user_id'])
        user_response = get_users()

# Run the script
if __name__ == "__main__"
    delete_users()

Exporting Orgs From Clerk

We now need to export your organization data. We can do so using Clerk's backend API. This script will create a file called org_data.json which we'll use in the next step to import into PropelAuth.

Get your Clerk API Secret key (found under "API Keys" in Clerk) and run this script to create a json containing all of your org and org membership data.

import json
import requests
from propelauth_py import init_base_auth

# Replace with your Clerk Secret key
CLERK_API_KEY="k9HZ5..."

# Replace with your PropelAuth Auth URL and API key
PROPELAUTH_AUTH_URL = "https://auth.yourdomain.com"
PROPELAUTH_AUTH_API_KEY = "b1988511..."

auth = init_base_auth(PROPELAUTH_AUTH_URL, PROPELAUTH_AUTH_API_KEY)

def get_propelauth_user(email):
    try:
        user = auth.fetch_user_metadata_by_email(email)
        return user['user_id']
    except requests.exceptions.RequestException as e:
        print(f"Error making API call to get propelauth user: {email}")
        print(f"Error: {e}")

def get_org_members(org_id):
    total_members = []
    more_members = True
    headers = {
        'Authorization': f'Bearer {CLERK_API_KEY}'
    }
    offset=0
    while more_members:
        request = requests.get(url=f"https://api.clerk.com/v1/organizations/{org_id}/memberships?limit=500&offset={offset}", headers=headers)
        response = request.json()
        org_members = response['data']
        if len(org_members) > 0:
            total_members += org_members
            offset += 500
        else:
            more_members = False
    for member in total_members:
        member['propelauth_user_id'] = get_propelauth_user(member['public_user_data']['identifier'])
        
    return total_members

def export_orgs():
    total_orgs = []
    more_orgs = True
    headers = {
        'Authorization': f'Bearer {CLERK_API_KEY}'
    }
    offset=0
    while more_orgs:
        request = requests.get(url=f"https://api.clerk.com/v1/organizations?limit=500&offset={offset}", headers=headers)
        response = request.json()
        orgs = response['data']
        if len(orgs) > 0:
            for org in orgs:
                org_members = get_org_members(org['id'])
                org['members'] = org_members
            total_orgs += orgs
            offset += 500
        else:
            more_orgs = False
            
    with open("org_data.json", "w") as outfile:
        json.dump(total_orgs, outfile)
        
if __name__ == "__main__":
    export_orgs()

Importing Orgs into PropelAuth

You should now have a file called org_data.json that contains all of your orgs, who belongs to each org, and which role the user belongs to for each org. Lets import this data using PropelAuth's API.

There are a variety of ways you might be handling org roles in Clerk. By default, this script will use the role found in the user's role property after removing org: from the beginning of it. You can change this by updating the map_clerk_role_to_propelauth_role function.

import requests
import json
from propelauth_py import init_base_auth

# Replace with your PropelAuth Auth URL and API key
PROPELAUTH_AUTH_URL = "https://auth.yourdomain.com"
PROPELAUTH_AUTH_API_KEY = "b1988511..."

# replace with your JSON path
ORG_JSON = "./org_data.json"

auth = init_base_auth(PROPELAUTH_AUTH_URL, PROPELAUTH_AUTH_API_KEY)

def map_clerk_role_to_propelauth_role(user_role):
    # implement me
    if user_role.startswith("org:"):
        user_role = user_role[4:] # remove "org:"
    return user_role.capitalize()

def create_propelauth_org(org_data):
    try:
        response = auth.create_org(
            name = org_data['name'],
            legacy_org_id = org_data['legacy_org_id']
        )
        print(f"API call successful to create org: {org_data['name']}")
        return response
    except requests.exceptions.RequestException as e:
        print(f"Error making API call to create org: {org_data['name']}")
        print(f"Error: {e}")

def add_user_to_org(org_id, org_user):
    try:
        auth.add_user_to_org(
            user_id=org_user['propelauth_user_id'],
            org_id=org_id,
            role=map_clerk_role_to_propelauth_role(org_user['role']),
        )
        print(f"API call successful to add user {org_id} to org {org_id}")
    except requests.exceptions.RequestException as e:
        print(f"Error making API call to add user {org_id} to org {org_id}")
        print(f"Error: {e}")

def process_orgs():
    with open(ORG_JSON, 'r') as infile:
        org_data = json.load(infile)

        for org in org_data:
            create_org_response = create_propelauth_org({
                'name': org['name'],
                'legacy_org_id': org['id']
            })
            org_id = create_org_response['org_id']
            for member in org['members']:
                add_user_to_org(org_id, member)

# Run the script
if __name__ == "__main__":
    process_orgs()

All of your orgs and users are now imported! If you run into any problems, do not hesitate to reach out to support@propelauth.com.