v/cmd/tools/vpm.v

// Copyright (c) 2019-2023 Alexander Medvednikov. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.
module main

import os
import rand
import os.cmdline
import net.http
import net.urllib
import json
import v.help
import v.vmod

const (
	default_vpm_server_urls   = ['https://vpm.vlang.io', 'https://vpm.url4e.com']
	vpm_server_urls           = rand.shuffle_clone(default_vpm_server_urls) or { [] } // ensure that all queries are distributed fairly
	valid_vpm_commands        = ['help', 'search', 'install', 'update', 'upgrade', 'outdated',
		'list', 'remove', 'show']
	excluded_dirs             = ['cache', 'vlib']
	supported_vcs_systems     = ['git', 'hg']
	supported_vcs_folders     = ['.git', '.hg']
	supported_vcs_update_cmds = {
		'git': 'git pull --recurse-submodules' // pulling with `--depth=1` leads to conflicts, when the upstream is more than 1 commit newer
		'hg':  'hg pull --update'
	}
	supported_vcs_install_cmds = {
		'git': 'git clone --depth=1 --recursive --shallow-submodules'
		'hg':  'hg clone'
	}
	supported_vcs_outdated_steps = {
		'git': ['git fetch', 'git rev-parse @', 'git rev-parse @{u}']
		'hg':  ['hg incoming']
	}
	supported_vcs_version_cmds = {
		'git': 'git version'
		'hg':  'hg version'
	}
)

struct Mod {
	id           int
	name         string
	url          string
	nr_downloads int
	vcs          string
}

struct Vmod {
mut:
	name    string
	version string
	deps    []string
}

enum Source {
	git
	hg
	vpm
}

fn main() {
	init_settings()
	// This tool is intended to be launched by the v frontend,
	// which provides the path to V inside os.getenv('VEXE')
	// args are: vpm [options] SUBCOMMAND module names
	params := cmdline.only_non_options(os.args[1..])
	options := cmdline.only_options(os.args[1..])
	verbose_println('cli params: ${params}')
	if params.len < 1 {
		vpm_help()
		exit(5)
	}
	vpm_command := params[0]
	mut module_names := params[1..]
	ensure_vmodules_dir_exist()
	// println('module names: ') println(module_names)
	match vpm_command {
		'help' {
			vpm_help()
		}
		'search' {
			vpm_search(module_names)
		}
		'install' {
			if module_names.len == 0 && os.exists('./v.mod') {
				println('Detected v.mod file inside the project directory. Using it...')
				manifest := vmod.from_file('./v.mod') or { panic(err) }
				module_names = manifest.dependencies.clone()
			}

			if '--once' in options {
				module_names = vpm_once_filter(module_names)

				if module_names.len == 0 {
					return
				}
			}

			external_module_names := module_names.filter(it.starts_with('https://'))
			vpm_module_names := module_names.filter(it !in external_module_names)

			if vpm_module_names.len > 0 {
				vpm_install(vpm_module_names, Source.vpm)
			}

			if external_module_names.len > 0 {
				mut external_source := Source.git

				if '--hg' in options {
					external_source = Source.hg
				}

				vpm_install(external_module_names, external_source)
			}
		}
		'update' {
			vpm_update(module_names)
		}
		'upgrade' {
			vpm_upgrade()
		}
		'outdated' {
			vpm_outdated()
		}
		'list' {
			vpm_list()
		}
		'remove' {
			vpm_remove(module_names)
		}
		'show' {
			vpm_show(module_names)
		}
		else {
			eprintln('Error: you tried to run "v ${vpm_command}"')
			eprintln('... but the v package management tool vpm only knows about these commands:')
			for validcmd in valid_vpm_commands {
				eprintln('    v ${validcmd}')
			}
			exit(3)
		}
	}
}

fn vpm_search(keywords []string) {
	search_keys := keywords.map(it.replace('_', '-'))
	if settings.is_help {
		help.print_and_exit('search')
		exit(0)
	}
	if search_keys.len == 0 {
		eprintln('´v search´ requires *at least one* keyword.')
		exit(2)
	}
	modules := get_all_modules()
	installed_modules := get_installed_modules()
	joined := search_keys.join(', ')
	mut index := 0
	for mod in modules {
		for k in search_keys {
			if !mod.contains(k) {
				continue
			}
			if index == 0 {
				println('Search results for "${joined}":\n')
			}
			index++
			mut parts := mod.split('.')
			// in case the author isn't present
			if parts.len == 1 {
				parts << parts[0]
				parts[0] = ' '
			} else {
				parts[0] = ' by ${parts[0]} '
			}
			installed := if mod in installed_modules { ' (installed)' } else { '' }
			println('${index}. ${parts[1]}${parts[0]}[${mod}]${installed}')
			break
		}
	}
	if index == 0 {
		vexe := os.getenv('VEXE')
		vroot := os.real_path(os.dir(vexe))
		mut messages := ['No module(s) found for `${joined}` .']
		for vlibmod in search_keys {
			if os.is_dir(os.join_path(vroot, 'vlib', vlibmod)) {
				messages << 'There is already an existing "${vlibmod}" module in vlib, so you can just `import ${vlibmod}` .'
			}
		}
		for m in messages {
			println(m)
		}
	} else {
		eprintln('\nUse "v install author_name.module_name" to install the module.')
	}
}

fn vpm_install_from_vpm(module_names []string) {
	mut errors := 0
	for n in module_names {
		name := n.trim_space().replace('_', '-')
		mod := get_module_meta_info(name) or {
			errors++
			eprintln('Errors while retrieving meta data for module ${name}:')
			eprintln(err)
			continue
		}
		mut vcs := mod.vcs
		if vcs == '' {
			vcs = supported_vcs_systems[0]
		}
		if vcs !in supported_vcs_systems {
			errors++
			eprintln('Skipping module "${name}", since it uses an unsupported VCS {${vcs}} .')
			continue
		}
		if !ensure_vcs_is_installed(vcs) {
			errors++
			eprintln('VPM needs `${vcs}` to be installed.')
			continue
		}
		//
		minfo := mod_name_info(mod.name)
		if os.exists(minfo.final_module_path) {
			vpm_update([name])
			continue
		}
		println('Installing module "${name}" from "${mod.url}" to "${minfo.final_module_path}" ...')
		increment_module_download_count(name) or {
			errors++
			eprintln('Errors while incrementing the download count for ${name}:')
		}
		vcs_install_cmd := supported_vcs_install_cmds[vcs]
		cmd := '${vcs_install_cmd} "${mod.url}" "${minfo.final_module_path}"'
		verbose_println('      command: ${cmd}')
		cmdres := os.execute(cmd)
		if cmdres.exit_code != 0 {
			errors++
			eprintln('Failed installing module "${name}" to "${minfo.final_module_path}" .')
			print_failed_cmd(cmd, cmdres)
			continue
		}
		resolve_dependencies(name, minfo.final_module_path, module_names)
	}
	if errors > 0 {
		exit(1)
	}
}

fn print_failed_cmd(cmd string, cmdres os.Result) {
	verbose_println('Failed command: ${cmd}')
	verbose_println('Failed command output:\n${cmdres.output}')
}

fn ensure_vcs_is_installed(vcs string) bool {
	mut res := true
	cmd := supported_vcs_version_cmds[vcs]
	cmdres := os.execute(cmd)
	if cmdres.exit_code != 0 {
		print_failed_cmd(cmd, cmdres)
		res = false
	}
	return res
}

fn vpm_install_from_vcs(module_names []string, vcs_key string) {
	mut errors := 0
	for n in module_names {
		url := n.trim_space()

		first_cut_pos := url.last_index('/') or {
			errors++
			eprintln('Errors while retrieving name for module "${url}" :')
			eprintln(err)
			continue
		}

		mod_name := url.substr(first_cut_pos + 1, url.len)

		second_cut_pos := url.substr(0, first_cut_pos).last_index('/') or {
			errors++
			eprintln('Errors while retrieving name for module "${url}" :')
			eprintln(err)
			continue
		}

		repo_name := url.substr(second_cut_pos + 1, first_cut_pos)
		mut name := os.join_path(repo_name, mod_name)
		mod_name_as_path := name.replace('-', '_').to_lower()
		mut final_module_path := os.real_path(os.join_path(settings.vmodules_path, mod_name_as_path))
		if os.exists(final_module_path) {
			vpm_update([name.replace('-', '_')])
			continue
		}
		if !ensure_vcs_is_installed(vcs_key) {
			errors++
			eprintln('VPM needs `${vcs_key}` to be installed.')
			continue
		}
		println('Installing module "${name}" from "${url}" to "${final_module_path}" ...')
		vcs_install_cmd := supported_vcs_install_cmds[vcs_key]
		cmd := '${vcs_install_cmd} "${url}" "${final_module_path}"'
		verbose_println('      command: ${cmd}')
		cmdres := os.execute(cmd)
		if cmdres.exit_code != 0 {
			errors++
			eprintln('Failed installing module "${name}" to "${final_module_path}" .')
			print_failed_cmd(cmd, cmdres)
			continue
		}
		vmod_path := os.join_path(final_module_path, 'v.mod')
		if os.exists(vmod_path) {
			data := os.read_file(vmod_path) or { return }
			vmod_ := parse_vmod(data) or {
				eprintln(err)
				return
			}
			minfo := mod_name_info(vmod_.name)
			if final_module_path != minfo.final_module_path {
				println('Relocating module from "${name}" to "${vmod_.name}" ( "${minfo.final_module_path}" ) ...')
				if os.exists(minfo.final_module_path) {
					eprintln('Warning module "${minfo.final_module_path}" already exsits!')
					eprintln('Removing module "${minfo.final_module_path}" ...')
					os.rmdir_all(minfo.final_module_path) or {
						errors++
						println('Errors while removing "${minfo.final_module_path}" :')
						println(err)
						continue
					}
				}
				os.mv(final_module_path, minfo.final_module_path) or {
					errors++
					eprintln('Errors while relocating module "${name}" :')
					eprintln(err)
					os.rmdir_all(final_module_path) or {
						errors++
						eprintln('Errors while removing "${final_module_path}" :')
						eprintln(err)
						continue
					}
					continue
				}
				println('Module "${name}" relocated to "${vmod_.name}" successfully.')
				final_module_path = minfo.final_module_path
			}
			name = vmod_.name
		}
		resolve_dependencies(name, final_module_path, module_names)
	}
	if errors > 0 {
		exit(1)
	}
}

fn vpm_once_filter(module_names []string) []string {
	installed_modules := get_installed_modules()
	mut toinstall := []string{}
	for mn in module_names {
		if mn !in installed_modules {
			toinstall << mn
		}
	}
	return toinstall
}

fn vpm_install(module_names []string, source Source) {
	if settings.is_help {
		help.print_and_exit('install')
		exit(0)
	}
	if module_names.len == 0 {
		eprintln('´v install´ requires *at least one* module name.')
		exit(2)
	}
	match source {
		.vpm {
			vpm_install_from_vpm(module_names)
		}
		.git {
			vpm_install_from_vcs(module_names, 'git')
		}
		.hg {
			vpm_install_from_vcs(module_names, 'hg')
		}
	}
}

fn vpm_update(m []string) {
	mut module_names := m.clone()
	if settings.is_help {
		help.print_and_exit('update')
		exit(0)
	}
	if module_names.len == 0 {
		module_names = get_installed_modules()
	}
	mut errors := 0
	for modulename in module_names {
		zname := url_to_module_name(modulename)
		final_module_path := valid_final_path_of_existing_module(modulename) or { continue }
		os.chdir(final_module_path) or {}
		println('Updating module "${zname}" in "${final_module_path}" ...')
		vcs := vcs_used_in_dir(final_module_path) or { continue }
		if !ensure_vcs_is_installed(vcs[0]) {
			errors++
			println('VPM needs `${vcs}` to be installed.')
			continue
		}
		vcs_cmd := supported_vcs_update_cmds[vcs[0]]
		verbose_println('    command: ${vcs_cmd}')
		vcs_res := os.execute('${vcs_cmd}')
		if vcs_res.exit_code != 0 {
			errors++
			println('Failed updating module "${zname}" in "${final_module_path}" .')
			print_failed_cmd(vcs_cmd, vcs_res)
			continue
		} else {
			verbose_println('    ${vcs_res.output.trim_space()}')
			increment_module_download_count(zname) or {
				errors++
				eprintln('Errors while incrementing the download count for ${zname}:')
			}
		}
		resolve_dependencies(modulename, final_module_path, module_names)
	}
	if errors > 0 {
		exit(1)
	}
}

fn get_outdated() ![]string {
	module_names := get_installed_modules()
	mut outdated := []string{}
	for name in module_names {
		final_module_path := valid_final_path_of_existing_module(name) or { continue }
		os.chdir(final_module_path) or {}
		vcs := vcs_used_in_dir(final_module_path) or { continue }
		vcs_cmd_steps := supported_vcs_outdated_steps[vcs[0]]
		mut outputs := []string{}
		for step in vcs_cmd_steps {
			res := os.execute(step)
			if res.exit_code < 0 {
				verbose_println('Error command: ${step}')
				verbose_println('Error details:\n${res.output}')
				return error('Error while checking latest commits for "${name}" .')
			}
			if vcs[0] == 'hg' {
				if res.exit_code == 1 {
					outdated << name
				}
			} else {
				outputs << res.output
			}
		}
		if vcs[0] == 'git' && outputs[1] != outputs[2] {
			outdated << name
		}
	}
	return outdated
}

fn vpm_upgrade() {
	outdated := get_outdated() or { exit(1) }
	if outdated.len > 0 {
		vpm_update(outdated)
	} else {
		println('Modules are up to date.')
	}
}

fn vpm_outdated() {
	outdated := get_outdated() or { exit(1) }
	if outdated.len > 0 {
		eprintln('Outdated modules:')
		for m in outdated {
			eprintln('  ${m}')
		}
	} else {
		println('Modules are up to date.')
	}
}

fn vpm_list() {
	module_names := get_installed_modules()
	if module_names.len == 0 {
		eprintln('You have no modules installed.')
		exit(0)
	}
	for mod in module_names {
		println(mod)
	}
}

fn vpm_remove(module_names []string) {
	if settings.is_help {
		help.print_and_exit('remove')
		exit(0)
	}
	if module_names.len == 0 {
		eprintln('´v remove´ requires *at least one* module name.')
		exit(2)
	}
	for name in module_names {
		final_module_path := valid_final_path_of_existing_module(name) or { continue }
		eprintln('Removing module "${name}" ...')
		verbose_println('removing folder ${final_module_path}')
		os.rmdir_all(final_module_path) or {
			verbose_println('error while removing "${final_module_path}": ${err.msg()}')
		}
		// delete author directory if it is empty
		author := name.split('.')[0]
		author_dir := os.real_path(os.join_path(settings.vmodules_path, author))
		if !os.exists(author_dir) {
			continue
		}
		if os.is_dir_empty(author_dir) {
			verbose_println('removing author folder ${author_dir}')
			os.rmdir(author_dir) or {
				verbose_println('error while removing "${author_dir}": ${err.msg()}')
			}
		}
	}
}

fn valid_final_path_of_existing_module(modulename string) ?string {
	name := if mod := get_mod_by_url(modulename) { mod.name } else { modulename }
	minfo := mod_name_info(name)
	if !os.exists(minfo.final_module_path) {
		eprintln('No module with name "${minfo.mname_normalised}" exists at ${minfo.final_module_path}')
		return none
	}
	if !os.is_dir(minfo.final_module_path) {
		eprintln('Skipping "${minfo.final_module_path}", since it is not a folder.')
		return none
	}
	vcs_used_in_dir(minfo.final_module_path) or {
		eprintln('Skipping "${minfo.final_module_path}", since it does not use a supported vcs.')
		return none
	}
	return minfo.final_module_path
}

fn ensure_vmodules_dir_exist() {
	if !os.is_dir(settings.vmodules_path) {
		println('Creating "${settings.vmodules_path}/" ...')
		os.mkdir(settings.vmodules_path) or { panic(err) }
	}
}

fn vpm_help() {
	help.print_and_exit('vpm')
}

fn vcs_used_in_dir(dir string) ?[]string {
	mut vcs := []string{}
	for repo_subfolder in supported_vcs_folders {
		checked_folder := os.real_path(os.join_path(dir, repo_subfolder))
		if os.is_dir(checked_folder) {
			vcs << repo_subfolder.replace('.', '')
		}
	}
	if vcs.len == 0 {
		return none
	}
	return vcs
}

fn get_installed_modules() []string {
	dirs := os.ls(settings.vmodules_path) or { return [] }
	mut modules := []string{}
	for dir in dirs {
		adir := os.join_path(settings.vmodules_path, dir)
		if dir in excluded_dirs || !os.is_dir(adir) {
			continue
		}
		if os.exists(os.join_path(adir, 'v.mod')) && os.exists(os.join_path(adir, '.git', 'config')) {
			// an official vlang module with a short module name, like `vsl`, `ui` or `markdown`
			modules << dir
			continue
		}
		author := dir
		mods := os.ls(adir) or { continue }
		for m in mods {
			vcs_used_in_dir(os.join_path(adir, m)) or { continue }
			modules << '${author}.${m}'
		}
	}
	return modules
}

struct ModNameInfo {
mut:
	mname             string // The-user.The-mod , *never* The-user.The-mod.git
	mname_normalised  string // the_user.the_mod
	mname_as_path     string // the_user/the_mod
	final_module_path string // ~/.vmodules/the_user/the_mod
}

fn mod_name_info(mod_name string) ModNameInfo {
	mut info := ModNameInfo{}
	info.mname = if mod_name.ends_with('.git') { mod_name.replace('.git', '') } else { mod_name }
	info.mname_normalised = info.mname.replace('-', '_').to_lower()
	info.mname_as_path = info.mname_normalised.replace('.', os.path_separator)
	info.final_module_path = os.real_path(os.join_path(settings.vmodules_path, info.mname_as_path))
	return info
}

fn url_to_module_name(modulename string) string {
	mut res := if mod := get_mod_by_url(modulename) { mod.name } else { modulename }
	if res.ends_with('.git') {
		res = res.replace('.git', '')
	}
	return res
}

fn get_all_modules() []string {
	url := get_working_server_url()
	r := http.get(url) or { panic(err) }
	if r.status_code != 200 {
		eprintln('Failed to search vpm.vlang.io. Status code: ${r.status_code}')
		exit(1)
	}
	s := r.body
	mut read_len := 0
	mut modules := []string{}
	for read_len < s.len {
		mut start_token := "<a href='/mod"
		end_token := '</a>'
		// get the start index of the module entry
		mut start_index := s.index_after(start_token, read_len)
		if start_index == -1 {
			start_token = '<a href="/mod'
			start_index = s.index_after(start_token, read_len)
			if start_index == -1 {
				break
			}
		}
		// get the index of the end of anchor (a) opening tag
		// we use the previous start_index to make sure we are getting a module and not just a random 'a' tag
		start_token = '>'
		start_index = s.index_after(start_token, start_index) + start_token.len

		// get the index of the end of module entry
		end_index := s.index_after(end_token, start_index)
		if end_index == -1 {
			break
		}
		modules << s[start_index..end_index]
		read_len = end_index
		if read_len >= s.len {
			break
		}
	}
	return modules
}

fn resolve_dependencies(name string, module_path string, module_names []string) {
	vmod_path := os.join_path(module_path, 'v.mod')
	if !os.exists(vmod_path) {
		return
	}
	data := os.read_file(vmod_path) or { return }
	vmod_ := parse_vmod(data) or {
		eprintln(err)
		return
	}
	mut deps := []string{}
	// filter out dependencies that were already specified by the user
	for d in vmod_.deps {
		if d !in module_names {
			deps << d
		}
	}
	if deps.len > 0 {
		println('Resolving ${deps.len} dependencies for module "${name}" ...')
		verbose_println('Found dependencies: ${deps}')
		vpm_install(deps, Source.vpm)
	}
}

fn parse_vmod(data string) !Vmod {
	manifest := vmod.decode(data) or { return error('Parsing v.mod file failed, ${err}') }
	mut vmod_ := Vmod{}
	vmod_.name = manifest.name
	vmod_.version = manifest.version
	vmod_.deps = manifest.dependencies
	return vmod_
}

fn get_working_server_url() string {
	server_urls := if settings.server_urls.len > 0 {
		settings.server_urls
	} else {
		vpm_server_urls
	}
	for url in server_urls {
		verbose_println('Trying server url: ${url}')
		http.head(url) or {
			verbose_println('                   ${url} failed.')
			continue
		}
		return url
	}
	panic('No responding vpm server found. Please check your network connectivity and try again later.')
}

// settings context:
struct VpmSettings {
mut:
	is_help       bool
	is_verbose    bool
	server_urls   []string
	vmodules_path string
}

const (
	settings = &VpmSettings{}
)

fn init_settings() {
	mut s := &VpmSettings(unsafe { nil })
	unsafe {
		s = settings
	}
	s.is_help = '-h' in os.args || '--help' in os.args || 'help' in os.args
	s.is_verbose = '-v' in os.args
	s.server_urls = cmdline.options(os.args, '-server-url')
	s.vmodules_path = os.vmodules_dir()
}

fn verbose_println(s string) {
	if settings.is_verbose {
		println(s)
	}
}

fn get_mod_by_url(name string) !Mod {
	if purl := urllib.parse(name) {
		verbose_println('purl: ${purl}')
		mod := Mod{
			name: purl.path.trim_left('/').trim_right('/').replace('/', '.')
			url: name
		}
		verbose_println(mod.str())
		return mod
	}
	return error('invalid url: ${name}')
}

fn get_module_meta_info(name string) !Mod {
	if mod := get_mod_by_url(name) {
		return mod
	}
	mut errors := []string{}

	for server_url in vpm_server_urls {
		modurl := server_url + '/api/packages/${name}'
		verbose_println('Retrieving module metadata from: "${modurl}" ...')
		r := http.get(modurl) or {
			errors << 'Http server did not respond to our request for "${modurl}" .'
			errors << 'Error details: ${err}'
			continue
		}
		if r.status_code == 404 || r.body.trim_space() == '404' {
			errors << 'Skipping module "${name}", since "${server_url}" reported that "${name}" does not exist.'
			continue
		}
		if r.status_code != 200 {
			errors << 'Skipping module "${name}", since "${server_url}" responded with ${r.status_code} http status code. Please try again later.'
			continue
		}
		s := r.body
		if s.len > 0 && s[0] != `{` {
			errors << 'Invalid json data'
			errors << s.trim_space().limit(100) + ' ...'
			continue
		}
		mod := json.decode(Mod, s) or {
			errors << 'Skipping module "${name}", since its information is not in json format.'
			continue
		}
		if '' == mod.url || '' == mod.name {
			errors << 'Skipping module "${name}", since it is missing name or url information.'
			continue
		}
		return mod
	}
	return error(errors.join_lines())
}

fn increment_module_download_count(name string) ! {
	mut errors := []string{}

	for server_url in vpm_server_urls {
		modurl := server_url + '/api/packages/${name}/incr_downloads'
		r := http.post(modurl, '') or {
			errors << 'Http server did not respond to our request for "${modurl}" .'
			errors << 'Error details: ${err}'
			continue
		}
		if r.status_code != 200 {
			errors << 'Failed to increment the download count for module "${name}", since "${server_url}" responded with ${r.status_code} http status code. Please try again later.'
			continue
		}
		return
	}
	return error(errors.join_lines())
}

fn vpm_show(module_names []string) {
	installed_modules := get_installed_modules()
	for module_name in module_names {
		if module_name !in installed_modules {
			module_meta_info := get_module_meta_info(module_name) or { continue }
			print('
Name: ${module_meta_info.name}
Homepage: ${module_meta_info.url}
Downloads: ${module_meta_info.nr_downloads}
Installed: False
--------
')
			continue
		}
		path := os.join_path(os.vmodules_dir(), module_name.replace('.', os.path_separator))
		mod := vmod.from_file(os.join_path(path, 'v.mod')) or { continue }
		print('Name: ${mod.name}
Version: ${mod.version}
Description: ${mod.description}
Homepage: ${mod.repo_url}
Author: ${mod.author}
License: ${mod.license}
Location: ${path}
Requires: ${mod.dependencies.join(', ')}
--------
')
	}
}